Making it easier to send arbitrary structures as work orders to MT workers
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard.
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of FFmpeg.
9  *
10  * FFmpeg is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * FFmpeg is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with FFmpeg; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file mpegvideo_enc.c
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include "avcodec.h"
31 #include "dsputil.h"
32 #include "mpegvideo.h"
33 #include "mpegvideo_common.h"
34 #include "mjpegenc.h"
35 #include "msmpeg4.h"
36 #include "h263.h"
37 #include "faandct.h"
38 #include <limits.h>
39
40 //#undef NDEBUG
41 //#include <assert.h>
42
43 static int encode_picture(MpegEncContext *s, int picture_number);
44 static int dct_quantize_refine(MpegEncContext *s, DCTELEM *block, int16_t *weight, DCTELEM *orig, int n, int qscale);
45 static int sse_mb(MpegEncContext *s);
46
47 /* enable all paranoid tests for rounding, overflows, etc... */
48 //#define PARANOID
49
50 //#define DEBUG
51
52 static const uint16_t aanscales[64] = {
53     /* precomputed values scaled up by 14 bits */
54     16384, 22725, 21407, 19266, 16384, 12873,  8867,  4520,
55     22725, 31521, 29692, 26722, 22725, 17855, 12299,  6270,
56     21407, 29692, 27969, 25172, 21407, 16819, 11585,  5906,
57     19266, 26722, 25172, 22654, 19266, 15137, 10426,  5315,
58     16384, 22725, 21407, 19266, 16384, 12873,  8867,  4520,
59     12873, 17855, 16819, 15137, 12873, 10114,  6967,  3552,
60     8867 , 12299, 11585, 10426,  8867,  6967,  4799,  2446,
61     4520 ,  6270,  5906,  5315,  4520,  3552,  2446,  1247
62 };
63
64 const uint16_t ff_inv_aanscales[64] = {
65   4096,  2953,  3135,  3483,  4096,  5213,  7568, 14846,
66   2953,  2129,  2260,  2511,  2953,  3759,  5457, 10703,
67   3135,  2260,  2399,  2666,  3135,  3990,  5793, 11363,
68   3483,  2511,  2666,  2962,  3483,  4433,  6436, 12625,
69   4096,  2953,  3135,  3483,  4096,  5213,  7568, 14846,
70   5213,  3759,  3990,  4433,  5213,  6635,  9633, 18895,
71   7568,  5457,  5793,  6436,  7568,  9633, 13985, 27432,
72  14846, 10703, 11363, 12625, 14846, 18895, 27432, 53809,
73 };
74
75 static uint8_t default_mv_penalty[MAX_FCODE+1][MAX_MV*2+1];
76 static uint8_t default_fcode_tab[MAX_MV*2+1];
77
78 void ff_convert_matrix(DSPContext *dsp, int (*qmat)[64], uint16_t (*qmat16)[2][64],
79                            const uint16_t *quant_matrix, int bias, int qmin, int qmax, int intra)
80 {
81     int qscale;
82     int shift=0;
83
84     for(qscale=qmin; qscale<=qmax; qscale++){
85         int i;
86         if (dsp->fdct == ff_jpeg_fdct_islow
87 #ifdef FAAN_POSTSCALE
88             || dsp->fdct == ff_faandct
89 #endif
90             ) {
91             for(i=0;i<64;i++) {
92                 const int j= dsp->idct_permutation[i];
93                 /* 16 <= qscale * quant_matrix[i] <= 7905 */
94                 /* 19952         <= aanscales[i] * qscale * quant_matrix[i]           <= 249205026 */
95                 /* (1<<36)/19952 >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= (1<<36)/249205026 */
96                 /* 3444240       >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= 275 */
97
98                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
99                                 (qscale * quant_matrix[j]));
100             }
101         } else if (dsp->fdct == fdct_ifast
102 #ifndef FAAN_POSTSCALE
103                    || dsp->fdct == ff_faandct
104 #endif
105                    ) {
106             for(i=0;i<64;i++) {
107                 const int j= dsp->idct_permutation[i];
108                 /* 16 <= qscale * quant_matrix[i] <= 7905 */
109                 /* 19952         <= aanscales[i] * qscale * quant_matrix[i]           <= 249205026 */
110                 /* (1<<36)/19952 >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= (1<<36)/249205026 */
111                 /* 3444240       >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= 275 */
112
113                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) /
114                                 (aanscales[i] * qscale * quant_matrix[j]));
115             }
116         } else {
117             for(i=0;i<64;i++) {
118                 const int j= dsp->idct_permutation[i];
119                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
120                    So 16           <= qscale * quant_matrix[i]             <= 7905
121                    so (1<<19) / 16 >= (1<<19) / (qscale * quant_matrix[i]) >= (1<<19) / 7905
122                    so 32768        >= (1<<19) / (qscale * quant_matrix[i]) >= 67
123                 */
124                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / (qscale * quant_matrix[j]));
125 //                qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) / (qscale * quant_matrix[i]);
126                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / (qscale * quant_matrix[j]);
127
128                 if(qmat16[qscale][0][i]==0 || qmat16[qscale][0][i]==128*256) qmat16[qscale][0][i]=128*256-1;
129                 qmat16[qscale][1][i]= ROUNDED_DIV(bias<<(16-QUANT_BIAS_SHIFT), qmat16[qscale][0][i]);
130             }
131         }
132
133         for(i=intra; i<64; i++){
134             int64_t max= 8191;
135             if (dsp->fdct == fdct_ifast
136 #ifndef FAAN_POSTSCALE
137                    || dsp->fdct == ff_faandct
138 #endif
139                    ) {
140                 max= (8191LL*aanscales[i]) >> 14;
141             }
142             while(((max * qmat[qscale][i]) >> shift) > INT_MAX){
143                 shift++;
144             }
145         }
146     }
147     if(shift){
148         av_log(NULL, AV_LOG_INFO, "Warning, QMAT_SHIFT is larger than %d, overflows possible\n", QMAT_SHIFT - shift);
149     }
150 }
151
152 static inline void update_qscale(MpegEncContext *s){
153     s->qscale= (s->lambda*139 + FF_LAMBDA_SCALE*64) >> (FF_LAMBDA_SHIFT + 7);
154     s->qscale= av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
155
156     s->lambda2= (s->lambda*s->lambda + FF_LAMBDA_SCALE/2) >> FF_LAMBDA_SHIFT;
157 }
158
159 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix){
160     int i;
161
162     if(matrix){
163         put_bits(pb, 1, 1);
164         for(i=0;i<64;i++) {
165             put_bits(pb, 8, matrix[ ff_zigzag_direct[i] ]);
166         }
167     }else
168         put_bits(pb, 1, 0);
169 }
170
171 static void copy_picture_attributes(MpegEncContext *s, AVFrame *dst, AVFrame *src){
172     int i;
173
174     dst->pict_type              = src->pict_type;
175     dst->quality                = src->quality;
176     dst->coded_picture_number   = src->coded_picture_number;
177     dst->display_picture_number = src->display_picture_number;
178 //    dst->reference              = src->reference;
179     dst->pts                    = src->pts;
180     dst->interlaced_frame       = src->interlaced_frame;
181     dst->top_field_first        = src->top_field_first;
182
183     if(s->avctx->me_threshold){
184         if(!src->motion_val[0])
185             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_val not set!\n");
186         if(!src->mb_type)
187             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.mb_type not set!\n");
188         if(!src->ref_index[0])
189             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.ref_index not set!\n");
190         if(src->motion_subsample_log2 != dst->motion_subsample_log2)
191             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_subsample_log2 doesn't match! (%d!=%d)\n",
192             src->motion_subsample_log2, dst->motion_subsample_log2);
193
194         memcpy(dst->mb_type, src->mb_type, s->mb_stride * s->mb_height * sizeof(dst->mb_type[0]));
195
196         for(i=0; i<2; i++){
197             int stride= ((16*s->mb_width )>>src->motion_subsample_log2) + 1;
198             int height= ((16*s->mb_height)>>src->motion_subsample_log2);
199
200             if(src->motion_val[i] && src->motion_val[i] != dst->motion_val[i]){
201                 memcpy(dst->motion_val[i], src->motion_val[i], 2*stride*height*sizeof(int16_t));
202             }
203             if(src->ref_index[i] && src->ref_index[i] != dst->ref_index[i]){
204                 memcpy(dst->ref_index[i], src->ref_index[i], s->b8_stride*2*s->mb_height*sizeof(int8_t));
205             }
206         }
207     }
208 }
209
210 static void update_duplicate_context_after_me(MpegEncContext *dst, MpegEncContext *src){
211 #define COPY(a) dst->a= src->a
212     COPY(pict_type);
213     COPY(current_picture);
214     COPY(f_code);
215     COPY(b_code);
216     COPY(qscale);
217     COPY(lambda);
218     COPY(lambda2);
219     COPY(picture_in_gop_number);
220     COPY(gop_picture_number);
221     COPY(frame_pred_frame_dct); //FIXME don't set in encode_header
222     COPY(progressive_frame); //FIXME don't set in encode_header
223     COPY(partitioned_frame); //FIXME don't set in encode_header
224 #undef COPY
225 }
226
227 /**
228  * sets the given MpegEncContext to defaults for encoding.
229  * the changed fields will not depend upon the prior state of the MpegEncContext.
230  */
231 static void MPV_encode_defaults(MpegEncContext *s){
232     int i;
233     MPV_common_defaults(s);
234
235     for(i=-16; i<16; i++){
236         default_fcode_tab[i + MAX_MV]= 1;
237     }
238     s->me.mv_penalty= default_mv_penalty;
239     s->fcode_tab= default_fcode_tab;
240 }
241
242 /* init video encoder */
243 av_cold int MPV_encode_init(AVCodecContext *avctx)
244 {
245     MpegEncContext *s = avctx->priv_data;
246     int i;
247     int chroma_h_shift, chroma_v_shift;
248
249     MPV_encode_defaults(s);
250
251     switch (avctx->codec_id) {
252     case CODEC_ID_MPEG2VIDEO:
253         if(avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P){
254             av_log(avctx, AV_LOG_ERROR, "only YUV420 and YUV422 are supported\n");
255             return -1;
256         }
257         break;
258     case CODEC_ID_LJPEG:
259     case CODEC_ID_MJPEG:
260         if(avctx->pix_fmt != PIX_FMT_YUVJ420P && avctx->pix_fmt != PIX_FMT_YUVJ422P &&
261            ((avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P) || avctx->strict_std_compliance>FF_COMPLIANCE_INOFFICIAL)){
262             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
263             return -1;
264         }
265         break;
266     default:
267         if(avctx->pix_fmt != PIX_FMT_YUV420P){
268             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
269             return -1;
270         }
271     }
272
273     switch (avctx->pix_fmt) {
274     case PIX_FMT_YUVJ422P:
275     case PIX_FMT_YUV422P:
276         s->chroma_format = CHROMA_422;
277         break;
278     case PIX_FMT_YUVJ420P:
279     case PIX_FMT_YUV420P:
280     default:
281         s->chroma_format = CHROMA_420;
282         break;
283     }
284
285     s->bit_rate = avctx->bit_rate;
286     s->width = avctx->width;
287     s->height = avctx->height;
288     if(avctx->gop_size > 600 && avctx->strict_std_compliance>FF_COMPLIANCE_EXPERIMENTAL){
289         av_log(avctx, AV_LOG_ERROR, "Warning keyframe interval too large! reducing it ...\n");
290         avctx->gop_size=600;
291     }
292     s->gop_size = avctx->gop_size;
293     s->avctx = avctx;
294     s->flags= avctx->flags;
295     s->flags2= avctx->flags2;
296     s->max_b_frames= avctx->max_b_frames;
297     s->codec_id= avctx->codec->id;
298     s->luma_elim_threshold  = avctx->luma_elim_threshold;
299     s->chroma_elim_threshold= avctx->chroma_elim_threshold;
300     s->strict_std_compliance= avctx->strict_std_compliance;
301     s->data_partitioning= avctx->flags & CODEC_FLAG_PART;
302     s->quarter_sample= (avctx->flags & CODEC_FLAG_QPEL)!=0;
303     s->mpeg_quant= avctx->mpeg_quant;
304     s->rtp_mode= !!avctx->rtp_payload_size;
305     s->intra_dc_precision= avctx->intra_dc_precision;
306     s->user_specified_pts = AV_NOPTS_VALUE;
307
308     if (s->gop_size <= 1) {
309         s->intra_only = 1;
310         s->gop_size = 12;
311     } else {
312         s->intra_only = 0;
313     }
314
315     s->me_method = avctx->me_method;
316
317     /* Fixed QSCALE */
318     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
319
320     s->adaptive_quant= (   s->avctx->lumi_masking
321                         || s->avctx->dark_masking
322                         || s->avctx->temporal_cplx_masking
323                         || s->avctx->spatial_cplx_masking
324                         || s->avctx->p_masking
325                         || s->avctx->border_masking
326                         || (s->flags&CODEC_FLAG_QP_RD))
327                        && !s->fixed_qscale;
328
329     s->obmc= !!(s->flags & CODEC_FLAG_OBMC);
330     s->loop_filter= !!(s->flags & CODEC_FLAG_LOOP_FILTER);
331     s->alternate_scan= !!(s->flags & CODEC_FLAG_ALT_SCAN);
332     s->intra_vlc_format= !!(s->flags2 & CODEC_FLAG2_INTRA_VLC);
333     s->q_scale_type= !!(s->flags2 & CODEC_FLAG2_NON_LINEAR_QUANT);
334
335     if(avctx->rc_max_rate && !avctx->rc_buffer_size){
336         av_log(avctx, AV_LOG_ERROR, "a vbv buffer size is needed, for encoding with a maximum bitrate\n");
337         return -1;
338     }
339
340     if(avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate){
341         av_log(avctx, AV_LOG_INFO, "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
342     }
343
344     if(avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate){
345         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
346         return -1;
347     }
348
349     if(avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate){
350         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
351         return -1;
352     }
353
354     if(avctx->rc_max_rate && avctx->rc_max_rate == avctx->bit_rate && avctx->rc_max_rate != avctx->rc_min_rate){
355         av_log(avctx, AV_LOG_INFO, "impossible bitrate constraints, this will fail\n");
356     }
357
358     if(avctx->rc_buffer_size && avctx->bit_rate*av_q2d(avctx->time_base) > avctx->rc_buffer_size){
359         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
360         return -1;
361     }
362
363     if(avctx->bit_rate*av_q2d(avctx->time_base) > avctx->bit_rate_tolerance){
364         av_log(avctx, AV_LOG_ERROR, "bitrate tolerance too small for bitrate\n");
365         return -1;
366     }
367
368     if(   s->avctx->rc_max_rate && s->avctx->rc_min_rate == s->avctx->rc_max_rate
369        && (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO)
370        && 90000LL * (avctx->rc_buffer_size-1) > s->avctx->rc_max_rate*0xFFFFLL){
371
372         av_log(avctx, AV_LOG_INFO, "Warning vbv_delay will be set to 0xFFFF (=VBR) as the specified vbv buffer is too large for the given bitrate!\n");
373     }
374
375     if((s->flags & CODEC_FLAG_4MV) && s->codec_id != CODEC_ID_MPEG4
376        && s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P && s->codec_id != CODEC_ID_FLV1){
377         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
378         return -1;
379     }
380
381     if(s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE){
382         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with simple mb decision\n");
383         return -1;
384     }
385
386     if(s->obmc && s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P){
387         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with H263(+)\n");
388         return -1;
389     }
390
391     if(s->quarter_sample && s->codec_id != CODEC_ID_MPEG4){
392         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
393         return -1;
394     }
395
396     if(s->data_partitioning && s->codec_id != CODEC_ID_MPEG4){
397         av_log(avctx, AV_LOG_ERROR, "data partitioning not supported by codec\n");
398         return -1;
399     }
400
401     if(s->max_b_frames && s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG1VIDEO && s->codec_id != CODEC_ID_MPEG2VIDEO){
402         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
403         return -1;
404     }
405
406     if((s->flags & (CODEC_FLAG_INTERLACED_DCT|CODEC_FLAG_INTERLACED_ME|CODEC_FLAG_ALT_SCAN))
407        && s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG2VIDEO){
408         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
409         return -1;
410     }
411
412     if(s->mpeg_quant && s->codec_id != CODEC_ID_MPEG4){ //FIXME mpeg2 uses that too
413         av_log(avctx, AV_LOG_ERROR, "mpeg2 style quantization not supported by codec\n");
414         return -1;
415     }
416
417     if((s->flags & CODEC_FLAG_CBP_RD) && !avctx->trellis){
418         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
419         return -1;
420     }
421
422     if((s->flags & CODEC_FLAG_QP_RD) && s->avctx->mb_decision != FF_MB_DECISION_RD){
423         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
424         return -1;
425     }
426
427     if(s->avctx->scenechange_threshold < 1000000000 && (s->flags & CODEC_FLAG_CLOSED_GOP)){
428         av_log(avctx, AV_LOG_ERROR, "closed gop with scene change detection are not supported yet, set threshold to 1000000000\n");
429         return -1;
430     }
431
432     if((s->flags2 & CODEC_FLAG2_INTRA_VLC) && s->codec_id != CODEC_ID_MPEG2VIDEO){
433         av_log(avctx, AV_LOG_ERROR, "intra vlc table not supported by codec\n");
434         return -1;
435     }
436
437     if(s->flags & CODEC_FLAG_LOW_DELAY){
438         if (s->codec_id != CODEC_ID_MPEG2VIDEO){
439             av_log(avctx, AV_LOG_ERROR, "low delay forcing is only available for mpeg2\n");
440             return -1;
441         }
442         if (s->max_b_frames != 0){
443             av_log(avctx, AV_LOG_ERROR, "b frames cannot be used with low delay\n");
444             return -1;
445         }
446     }
447
448     if(s->q_scale_type == 1){
449         if(s->codec_id != CODEC_ID_MPEG2VIDEO){
450             av_log(avctx, AV_LOG_ERROR, "non linear quant is only available for mpeg2\n");
451             return -1;
452         }
453         if(avctx->qmax > 12){
454             av_log(avctx, AV_LOG_ERROR, "non linear quant only supports qmax <= 12 currently\n");
455             return -1;
456         }
457     }
458
459     if(s->avctx->thread_count > 1 && s->codec_id != CODEC_ID_MPEG4
460        && s->codec_id != CODEC_ID_MPEG1VIDEO && s->codec_id != CODEC_ID_MPEG2VIDEO
461        && (s->codec_id != CODEC_ID_H263P || !(s->flags & CODEC_FLAG_H263P_SLICE_STRUCT))){
462         av_log(avctx, AV_LOG_ERROR, "multi threaded encoding not supported by codec\n");
463         return -1;
464     }
465
466     if(s->avctx->thread_count > 1)
467         s->rtp_mode= 1;
468
469     if(!avctx->time_base.den || !avctx->time_base.num){
470         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
471         return -1;
472     }
473
474     i= (INT_MAX/2+128)>>8;
475     if(avctx->me_threshold >= i){
476         av_log(avctx, AV_LOG_ERROR, "me_threshold too large, max is %d\n", i - 1);
477         return -1;
478     }
479     if(avctx->mb_threshold >= i){
480         av_log(avctx, AV_LOG_ERROR, "mb_threshold too large, max is %d\n", i - 1);
481         return -1;
482     }
483
484     if(avctx->b_frame_strategy && (avctx->flags&CODEC_FLAG_PASS2)){
485         av_log(avctx, AV_LOG_INFO, "notice: b_frame_strategy only affects the first pass\n");
486         avctx->b_frame_strategy = 0;
487     }
488
489     i= ff_gcd(avctx->time_base.den, avctx->time_base.num);
490     if(i > 1){
491         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
492         avctx->time_base.den /= i;
493         avctx->time_base.num /= i;
494 //        return -1;
495     }
496
497     if(s->codec_id==CODEC_ID_MJPEG){
498         s->intra_quant_bias= 1<<(QUANT_BIAS_SHIFT-1); //(a + x/2)/x
499         s->inter_quant_bias= 0;
500     }else if(s->mpeg_quant || s->codec_id==CODEC_ID_MPEG1VIDEO || s->codec_id==CODEC_ID_MPEG2VIDEO){
501         s->intra_quant_bias= 3<<(QUANT_BIAS_SHIFT-3); //(a + x*3/8)/x
502         s->inter_quant_bias= 0;
503     }else{
504         s->intra_quant_bias=0;
505         s->inter_quant_bias=-(1<<(QUANT_BIAS_SHIFT-2)); //(a - x/4)/x
506     }
507
508     if(avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
509         s->intra_quant_bias= avctx->intra_quant_bias;
510     if(avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
511         s->inter_quant_bias= avctx->inter_quant_bias;
512
513     avcodec_get_chroma_sub_sample(avctx->pix_fmt, &chroma_h_shift, &chroma_v_shift);
514
515     if(avctx->codec_id == CODEC_ID_MPEG4 && s->avctx->time_base.den > (1<<16)-1){
516         av_log(avctx, AV_LOG_ERROR, "timebase not supported by mpeg 4 standard\n");
517         return -1;
518     }
519     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
520
521     switch(avctx->codec->id) {
522     case CODEC_ID_MPEG1VIDEO:
523         s->out_format = FMT_MPEG1;
524         s->low_delay= !!(s->flags & CODEC_FLAG_LOW_DELAY);
525         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
526         break;
527     case CODEC_ID_MPEG2VIDEO:
528         s->out_format = FMT_MPEG1;
529         s->low_delay= !!(s->flags & CODEC_FLAG_LOW_DELAY);
530         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
531         s->rtp_mode= 1;
532         break;
533     case CODEC_ID_LJPEG:
534     case CODEC_ID_MJPEG:
535         s->out_format = FMT_MJPEG;
536         s->intra_only = 1; /* force intra only for jpeg */
537         s->mjpeg_vsample[0] = 2;
538         s->mjpeg_vsample[1] = 2>>chroma_v_shift;
539         s->mjpeg_vsample[2] = 2>>chroma_v_shift;
540         s->mjpeg_hsample[0] = 2;
541         s->mjpeg_hsample[1] = 2>>chroma_h_shift;
542         s->mjpeg_hsample[2] = 2>>chroma_h_shift;
543         if (!(ENABLE_MJPEG_ENCODER || ENABLE_LJPEG_ENCODER)
544             || ff_mjpeg_encode_init(s) < 0)
545             return -1;
546         avctx->delay=0;
547         s->low_delay=1;
548         break;
549     case CODEC_ID_H261:
550         if (!ENABLE_H261_ENCODER)  return -1;
551         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
552             av_log(avctx, AV_LOG_ERROR, "The specified picture size of %dx%d is not valid for the H.261 codec.\nValid sizes are 176x144, 352x288\n", s->width, s->height);
553             return -1;
554         }
555         s->out_format = FMT_H261;
556         avctx->delay=0;
557         s->low_delay=1;
558         break;
559     case CODEC_ID_H263:
560         if (!ENABLE_H263_ENCODER)  return -1;
561         if (h263_get_picture_format(s->width, s->height) == 7) {
562             av_log(avctx, AV_LOG_INFO, "The specified picture size of %dx%d is not valid for the H.263 codec.\nValid sizes are 128x96, 176x144, 352x288, 704x576, and 1408x1152. Try H.263+.\n", s->width, s->height);
563             return -1;
564         }
565         s->out_format = FMT_H263;
566         s->obmc= (avctx->flags & CODEC_FLAG_OBMC) ? 1:0;
567         avctx->delay=0;
568         s->low_delay=1;
569         break;
570     case CODEC_ID_H263P:
571         s->out_format = FMT_H263;
572         s->h263_plus = 1;
573         /* Fx */
574         s->umvplus = (avctx->flags & CODEC_FLAG_H263P_UMV) ? 1:0;
575         s->h263_aic= (avctx->flags & CODEC_FLAG_AC_PRED) ? 1:0;
576         s->modified_quant= s->h263_aic;
577         s->alt_inter_vlc= (avctx->flags & CODEC_FLAG_H263P_AIV) ? 1:0;
578         s->obmc= (avctx->flags & CODEC_FLAG_OBMC) ? 1:0;
579         s->loop_filter= (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1:0;
580         s->unrestricted_mv= s->obmc || s->loop_filter || s->umvplus;
581         s->h263_slice_structured= (s->flags & CODEC_FLAG_H263P_SLICE_STRUCT) ? 1:0;
582
583         /* /Fx */
584         /* These are just to be sure */
585         avctx->delay=0;
586         s->low_delay=1;
587         break;
588     case CODEC_ID_FLV1:
589         s->out_format = FMT_H263;
590         s->h263_flv = 2; /* format = 1; 11-bit codes */
591         s->unrestricted_mv = 1;
592         s->rtp_mode=0; /* don't allow GOB */
593         avctx->delay=0;
594         s->low_delay=1;
595         break;
596     case CODEC_ID_RV10:
597         s->out_format = FMT_H263;
598         avctx->delay=0;
599         s->low_delay=1;
600         break;
601     case CODEC_ID_RV20:
602         s->out_format = FMT_H263;
603         avctx->delay=0;
604         s->low_delay=1;
605         s->modified_quant=1;
606         s->h263_aic=1;
607         s->h263_plus=1;
608         s->loop_filter=1;
609         s->unrestricted_mv= s->obmc || s->loop_filter || s->umvplus;
610         break;
611     case CODEC_ID_MPEG4:
612         s->out_format = FMT_H263;
613         s->h263_pred = 1;
614         s->unrestricted_mv = 1;
615         s->low_delay= s->max_b_frames ? 0 : 1;
616         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
617         break;
618     case CODEC_ID_MSMPEG4V1:
619         s->out_format = FMT_H263;
620         s->h263_msmpeg4 = 1;
621         s->h263_pred = 1;
622         s->unrestricted_mv = 1;
623         s->msmpeg4_version= 1;
624         avctx->delay=0;
625         s->low_delay=1;
626         break;
627     case CODEC_ID_MSMPEG4V2:
628         s->out_format = FMT_H263;
629         s->h263_msmpeg4 = 1;
630         s->h263_pred = 1;
631         s->unrestricted_mv = 1;
632         s->msmpeg4_version= 2;
633         avctx->delay=0;
634         s->low_delay=1;
635         break;
636     case CODEC_ID_MSMPEG4V3:
637         s->out_format = FMT_H263;
638         s->h263_msmpeg4 = 1;
639         s->h263_pred = 1;
640         s->unrestricted_mv = 1;
641         s->msmpeg4_version= 3;
642         s->flipflop_rounding=1;
643         avctx->delay=0;
644         s->low_delay=1;
645         break;
646     case CODEC_ID_WMV1:
647         s->out_format = FMT_H263;
648         s->h263_msmpeg4 = 1;
649         s->h263_pred = 1;
650         s->unrestricted_mv = 1;
651         s->msmpeg4_version= 4;
652         s->flipflop_rounding=1;
653         avctx->delay=0;
654         s->low_delay=1;
655         break;
656     case CODEC_ID_WMV2:
657         s->out_format = FMT_H263;
658         s->h263_msmpeg4 = 1;
659         s->h263_pred = 1;
660         s->unrestricted_mv = 1;
661         s->msmpeg4_version= 5;
662         s->flipflop_rounding=1;
663         avctx->delay=0;
664         s->low_delay=1;
665         break;
666     default:
667         return -1;
668     }
669
670     avctx->has_b_frames= !s->low_delay;
671
672     s->encoding = 1;
673
674     /* init */
675     if (MPV_common_init(s) < 0)
676         return -1;
677
678     if(!s->dct_quantize)
679         s->dct_quantize = dct_quantize_c;
680     if(!s->denoise_dct)
681         s->denoise_dct = denoise_dct_c;
682     s->fast_dct_quantize = s->dct_quantize;
683     if(avctx->trellis)
684         s->dct_quantize = dct_quantize_trellis_c;
685
686     if((ENABLE_H263P_ENCODER || ENABLE_RV20_ENCODER) && s->modified_quant)
687         s->chroma_qscale_table= ff_h263_chroma_qscale_table;
688     s->progressive_frame=
689     s->progressive_sequence= !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT|CODEC_FLAG_INTERLACED_ME|CODEC_FLAG_ALT_SCAN));
690     s->quant_precision=5;
691
692     ff_set_cmp(&s->dsp, s->dsp.ildct_cmp, s->avctx->ildct_cmp);
693     ff_set_cmp(&s->dsp, s->dsp.frame_skip_cmp, s->avctx->frame_skip_cmp);
694
695     if (ENABLE_H261_ENCODER && s->out_format == FMT_H261)
696         ff_h261_encode_init(s);
697     if (ENABLE_ANY_H263_ENCODER && s->out_format == FMT_H263)
698         h263_encode_init(s);
699     if (ENABLE_MSMPEG4_ENCODER && s->msmpeg4_version)
700         ff_msmpeg4_encode_init(s);
701     if ((ENABLE_MPEG1VIDEO_ENCODER || ENABLE_MPEG2VIDEO_ENCODER)
702         && s->out_format == FMT_MPEG1)
703         ff_mpeg1_encode_init(s);
704
705     /* init q matrix */
706     for(i=0;i<64;i++) {
707         int j= s->dsp.idct_permutation[i];
708         if(ENABLE_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4 && s->mpeg_quant){
709             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
710             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
711         }else if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
712             s->intra_matrix[j] =
713             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
714         }else
715         { /* mpeg1/2 */
716             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
717             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
718         }
719         if(s->avctx->intra_matrix)
720             s->intra_matrix[j] = s->avctx->intra_matrix[i];
721         if(s->avctx->inter_matrix)
722             s->inter_matrix[j] = s->avctx->inter_matrix[i];
723     }
724
725     /* precompute matrix */
726     /* for mjpeg, we do include qscale in the matrix */
727     if (s->out_format != FMT_MJPEG) {
728         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
729                        s->intra_matrix, s->intra_quant_bias, avctx->qmin, 31, 1);
730         ff_convert_matrix(&s->dsp, s->q_inter_matrix, s->q_inter_matrix16,
731                        s->inter_matrix, s->inter_quant_bias, avctx->qmin, 31, 0);
732     }
733
734     if(ff_rate_control_init(s) < 0)
735         return -1;
736
737     return 0;
738 }
739
740 av_cold int MPV_encode_end(AVCodecContext *avctx)
741 {
742     MpegEncContext *s = avctx->priv_data;
743
744     ff_rate_control_uninit(s);
745
746     MPV_common_end(s);
747     if ((ENABLE_MJPEG_ENCODER || ENABLE_LJPEG_ENCODER) && s->out_format == FMT_MJPEG)
748         ff_mjpeg_encode_close(s);
749
750     av_freep(&avctx->extradata);
751
752     return 0;
753 }
754
755 static int get_sae(uint8_t *src, int ref, int stride){
756     int x,y;
757     int acc=0;
758
759     for(y=0; y<16; y++){
760         for(x=0; x<16; x++){
761             acc+= FFABS(src[x+y*stride] - ref);
762         }
763     }
764
765     return acc;
766 }
767
768 static int get_intra_count(MpegEncContext *s, uint8_t *src, uint8_t *ref, int stride){
769     int x, y, w, h;
770     int acc=0;
771
772     w= s->width &~15;
773     h= s->height&~15;
774
775     for(y=0; y<h; y+=16){
776         for(x=0; x<w; x+=16){
777             int offset= x + y*stride;
778             int sad = s->dsp.sad[0](NULL, src + offset, ref + offset, stride, 16);
779             int mean= (s->dsp.pix_sum(src + offset, stride) + 128)>>8;
780             int sae = get_sae(src + offset, mean, stride);
781
782             acc+= sae + 500 < sad;
783         }
784     }
785     return acc;
786 }
787
788
789 static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg){
790     AVFrame *pic=NULL;
791     int64_t pts;
792     int i;
793     const int encoding_delay= s->max_b_frames;
794     int direct=1;
795
796     if(pic_arg){
797         pts= pic_arg->pts;
798         pic_arg->display_picture_number= s->input_picture_number++;
799
800         if(pts != AV_NOPTS_VALUE){
801             if(s->user_specified_pts != AV_NOPTS_VALUE){
802                 int64_t time= pts;
803                 int64_t last= s->user_specified_pts;
804
805                 if(time <= last){
806                     av_log(s->avctx, AV_LOG_ERROR, "Error, Invalid timestamp=%"PRId64", last=%"PRId64"\n", pts, s->user_specified_pts);
807                     return -1;
808                 }
809             }
810             s->user_specified_pts= pts;
811         }else{
812             if(s->user_specified_pts != AV_NOPTS_VALUE){
813                 s->user_specified_pts=
814                 pts= s->user_specified_pts + 1;
815                 av_log(s->avctx, AV_LOG_INFO, "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n", pts);
816             }else{
817                 pts= pic_arg->display_picture_number;
818             }
819         }
820     }
821
822   if(pic_arg){
823     if(encoding_delay && !(s->flags&CODEC_FLAG_INPUT_PRESERVED)) direct=0;
824     if(pic_arg->linesize[0] != s->linesize) direct=0;
825     if(pic_arg->linesize[1] != s->uvlinesize) direct=0;
826     if(pic_arg->linesize[2] != s->uvlinesize) direct=0;
827
828 //    av_log(AV_LOG_DEBUG, "%d %d %d %d\n",pic_arg->linesize[0], pic_arg->linesize[1], s->linesize, s->uvlinesize);
829
830     if(direct){
831         i= ff_find_unused_picture(s, 1);
832
833         pic= (AVFrame*)&s->picture[i];
834         pic->reference= 3;
835
836         for(i=0; i<4; i++){
837             pic->data[i]= pic_arg->data[i];
838             pic->linesize[i]= pic_arg->linesize[i];
839         }
840         alloc_picture(s, (Picture*)pic, 1);
841     }else{
842         i= ff_find_unused_picture(s, 0);
843
844         pic= (AVFrame*)&s->picture[i];
845         pic->reference= 3;
846
847         alloc_picture(s, (Picture*)pic, 0);
848
849         if(   pic->data[0] + INPLACE_OFFSET == pic_arg->data[0]
850            && pic->data[1] + INPLACE_OFFSET == pic_arg->data[1]
851            && pic->data[2] + INPLACE_OFFSET == pic_arg->data[2]){
852        // empty
853         }else{
854             int h_chroma_shift, v_chroma_shift;
855             avcodec_get_chroma_sub_sample(s->avctx->pix_fmt, &h_chroma_shift, &v_chroma_shift);
856
857             for(i=0; i<3; i++){
858                 int src_stride= pic_arg->linesize[i];
859                 int dst_stride= i ? s->uvlinesize : s->linesize;
860                 int h_shift= i ? h_chroma_shift : 0;
861                 int v_shift= i ? v_chroma_shift : 0;
862                 int w= s->width >>h_shift;
863                 int h= s->height>>v_shift;
864                 uint8_t *src= pic_arg->data[i];
865                 uint8_t *dst= pic->data[i];
866
867                 if(!s->avctx->rc_buffer_size)
868                     dst +=INPLACE_OFFSET;
869
870                 if(src_stride==dst_stride)
871                     memcpy(dst, src, src_stride*h);
872                 else{
873                     while(h--){
874                         memcpy(dst, src, w);
875                         dst += dst_stride;
876                         src += src_stride;
877                     }
878                 }
879             }
880         }
881     }
882     copy_picture_attributes(s, pic, pic_arg);
883     pic->pts= pts; //we set this here to avoid modifiying pic_arg
884   }
885
886     /* shift buffer entries */
887     for(i=1; i<MAX_PICTURE_COUNT /*s->encoding_delay+1*/; i++)
888         s->input_picture[i-1]= s->input_picture[i];
889
890     s->input_picture[encoding_delay]= (Picture*)pic;
891
892     return 0;
893 }
894
895 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref){
896     int x, y, plane;
897     int score=0;
898     int64_t score64=0;
899
900     for(plane=0; plane<3; plane++){
901         const int stride= p->linesize[plane];
902         const int bw= plane ? 1 : 2;
903         for(y=0; y<s->mb_height*bw; y++){
904             for(x=0; x<s->mb_width*bw; x++){
905                 int off= p->type == FF_BUFFER_TYPE_SHARED ? 0: 16;
906                 int v= s->dsp.frame_skip_cmp[1](s, p->data[plane] + 8*(x + y*stride)+off, ref->data[plane] + 8*(x + y*stride), stride, 8);
907
908                 switch(s->avctx->frame_skip_exp){
909                     case 0: score= FFMAX(score, v); break;
910                     case 1: score+= FFABS(v);break;
911                     case 2: score+= v*v;break;
912                     case 3: score64+= FFABS(v*v*(int64_t)v);break;
913                     case 4: score64+= v*v*(int64_t)(v*v);break;
914                 }
915             }
916         }
917     }
918
919     if(score) score64= score;
920
921     if(score64 < s->avctx->frame_skip_threshold)
922         return 1;
923     if(score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda)>>8))
924         return 1;
925     return 0;
926 }
927
928 static int estimate_best_b_count(MpegEncContext *s){
929     AVCodec *codec= avcodec_find_encoder(s->avctx->codec_id);
930     AVCodecContext *c= avcodec_alloc_context();
931     AVFrame input[FF_MAX_B_FRAMES+2];
932     const int scale= s->avctx->brd_scale;
933     int i, j, out_size, p_lambda, b_lambda, lambda2;
934     int outbuf_size= s->width * s->height; //FIXME
935     uint8_t *outbuf= av_malloc(outbuf_size);
936     int64_t best_rd= INT64_MAX;
937     int best_b_count= -1;
938
939     assert(scale>=0 && scale <=3);
940
941 //    emms_c();
942     p_lambda= s->last_lambda_for[FF_P_TYPE]; //s->next_picture_ptr->quality;
943     b_lambda= s->last_lambda_for[FF_B_TYPE]; //p_lambda *FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
944     if(!b_lambda) b_lambda= p_lambda; //FIXME we should do this somewhere else
945     lambda2= (b_lambda*b_lambda + (1<<FF_LAMBDA_SHIFT)/2 ) >> FF_LAMBDA_SHIFT;
946
947     c->width = s->width >> scale;
948     c->height= s->height>> scale;
949     c->flags= CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR | CODEC_FLAG_INPUT_PRESERVED /*| CODEC_FLAG_EMU_EDGE*/;
950     c->flags|= s->avctx->flags & CODEC_FLAG_QPEL;
951     c->mb_decision= s->avctx->mb_decision;
952     c->me_cmp= s->avctx->me_cmp;
953     c->mb_cmp= s->avctx->mb_cmp;
954     c->me_sub_cmp= s->avctx->me_sub_cmp;
955     c->pix_fmt = PIX_FMT_YUV420P;
956     c->time_base= s->avctx->time_base;
957     c->max_b_frames= s->max_b_frames;
958
959     if (avcodec_open(c, codec) < 0)
960         return -1;
961
962     for(i=0; i<s->max_b_frames+2; i++){
963         int ysize= c->width*c->height;
964         int csize= (c->width/2)*(c->height/2);
965         Picture pre_input, *pre_input_ptr= i ? s->input_picture[i-1] : s->next_picture_ptr;
966
967         avcodec_get_frame_defaults(&input[i]);
968         input[i].data[0]= av_malloc(ysize + 2*csize);
969         input[i].data[1]= input[i].data[0] + ysize;
970         input[i].data[2]= input[i].data[1] + csize;
971         input[i].linesize[0]= c->width;
972         input[i].linesize[1]=
973         input[i].linesize[2]= c->width/2;
974
975         if(pre_input_ptr && (!i || s->input_picture[i-1])) {
976             pre_input= *pre_input_ptr;
977
978             if(pre_input.type != FF_BUFFER_TYPE_SHARED && i) {
979                 pre_input.data[0]+=INPLACE_OFFSET;
980                 pre_input.data[1]+=INPLACE_OFFSET;
981                 pre_input.data[2]+=INPLACE_OFFSET;
982             }
983
984             s->dsp.shrink[scale](input[i].data[0], input[i].linesize[0], pre_input.data[0], pre_input.linesize[0], c->width, c->height);
985             s->dsp.shrink[scale](input[i].data[1], input[i].linesize[1], pre_input.data[1], pre_input.linesize[1], c->width>>1, c->height>>1);
986             s->dsp.shrink[scale](input[i].data[2], input[i].linesize[2], pre_input.data[2], pre_input.linesize[2], c->width>>1, c->height>>1);
987         }
988     }
989
990     for(j=0; j<s->max_b_frames+1; j++){
991         int64_t rd=0;
992
993         if(!s->input_picture[j])
994             break;
995
996         c->error[0]= c->error[1]= c->error[2]= 0;
997
998         input[0].pict_type= FF_I_TYPE;
999         input[0].quality= 1 * FF_QP2LAMBDA;
1000         out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[0]);
1001 //        rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1002
1003         for(i=0; i<s->max_b_frames+1; i++){
1004             int is_p= i % (j+1) == j || i==s->max_b_frames;
1005
1006             input[i+1].pict_type= is_p ? FF_P_TYPE : FF_B_TYPE;
1007             input[i+1].quality= is_p ? p_lambda : b_lambda;
1008             out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[i+1]);
1009             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1010         }
1011
1012         /* get the delayed frames */
1013         while(out_size){
1014             out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL);
1015             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1016         }
1017
1018         rd += c->error[0] + c->error[1] + c->error[2];
1019
1020         if(rd < best_rd){
1021             best_rd= rd;
1022             best_b_count= j;
1023         }
1024     }
1025
1026     av_freep(&outbuf);
1027     avcodec_close(c);
1028     av_freep(&c);
1029
1030     for(i=0; i<s->max_b_frames+2; i++){
1031         av_freep(&input[i].data[0]);
1032     }
1033
1034     return best_b_count;
1035 }
1036
1037 static void select_input_picture(MpegEncContext *s){
1038     int i;
1039
1040     for(i=1; i<MAX_PICTURE_COUNT; i++)
1041         s->reordered_input_picture[i-1]= s->reordered_input_picture[i];
1042     s->reordered_input_picture[MAX_PICTURE_COUNT-1]= NULL;
1043
1044     /* set next picture type & ordering */
1045     if(s->reordered_input_picture[0]==NULL && s->input_picture[0]){
1046         if(/*s->picture_in_gop_number >= s->gop_size ||*/ s->next_picture_ptr==NULL || s->intra_only){
1047             s->reordered_input_picture[0]= s->input_picture[0];
1048             s->reordered_input_picture[0]->pict_type= FF_I_TYPE;
1049             s->reordered_input_picture[0]->coded_picture_number= s->coded_picture_number++;
1050         }else{
1051             int b_frames;
1052
1053             if(s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor){
1054                 if(s->picture_in_gop_number < s->gop_size && skip_check(s, s->input_picture[0], s->next_picture_ptr)){
1055                 //FIXME check that te gop check above is +-1 correct
1056 //av_log(NULL, AV_LOG_DEBUG, "skip %p %"PRId64"\n", s->input_picture[0]->data[0], s->input_picture[0]->pts);
1057
1058                     if(s->input_picture[0]->type == FF_BUFFER_TYPE_SHARED){
1059                         for(i=0; i<4; i++)
1060                             s->input_picture[0]->data[i]= NULL;
1061                         s->input_picture[0]->type= 0;
1062                     }else{
1063                         assert(   s->input_picture[0]->type==FF_BUFFER_TYPE_USER
1064                                || s->input_picture[0]->type==FF_BUFFER_TYPE_INTERNAL);
1065
1066                         s->avctx->release_buffer(s->avctx, (AVFrame*)s->input_picture[0]);
1067                     }
1068
1069                     emms_c();
1070                     ff_vbv_update(s, 0);
1071
1072                     goto no_output_pic;
1073                 }
1074             }
1075
1076             if(s->flags&CODEC_FLAG_PASS2){
1077                 for(i=0; i<s->max_b_frames+1; i++){
1078                     int pict_num= s->input_picture[0]->display_picture_number + i;
1079
1080                     if(pict_num >= s->rc_context.num_entries)
1081                         break;
1082                     if(!s->input_picture[i]){
1083                         s->rc_context.entry[pict_num-1].new_pict_type = FF_P_TYPE;
1084                         break;
1085                     }
1086
1087                     s->input_picture[i]->pict_type=
1088                         s->rc_context.entry[pict_num].new_pict_type;
1089                 }
1090             }
1091
1092             if(s->avctx->b_frame_strategy==0){
1093                 b_frames= s->max_b_frames;
1094                 while(b_frames && !s->input_picture[b_frames]) b_frames--;
1095             }else if(s->avctx->b_frame_strategy==1){
1096                 for(i=1; i<s->max_b_frames+1; i++){
1097                     if(s->input_picture[i] && s->input_picture[i]->b_frame_score==0){
1098                         s->input_picture[i]->b_frame_score=
1099                             get_intra_count(s, s->input_picture[i  ]->data[0],
1100                                                s->input_picture[i-1]->data[0], s->linesize) + 1;
1101                     }
1102                 }
1103                 for(i=0; i<s->max_b_frames+1; i++){
1104                     if(s->input_picture[i]==NULL || s->input_picture[i]->b_frame_score - 1 > s->mb_num/s->avctx->b_sensitivity) break;
1105                 }
1106
1107                 b_frames= FFMAX(0, i-1);
1108
1109                 /* reset scores */
1110                 for(i=0; i<b_frames+1; i++){
1111                     s->input_picture[i]->b_frame_score=0;
1112                 }
1113             }else if(s->avctx->b_frame_strategy==2){
1114                 b_frames= estimate_best_b_count(s);
1115             }else{
1116                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1117                 b_frames=0;
1118             }
1119
1120             emms_c();
1121 //static int b_count=0;
1122 //b_count+= b_frames;
1123 //av_log(s->avctx, AV_LOG_DEBUG, "b_frames: %d\n", b_count);
1124
1125             for(i= b_frames - 1; i>=0; i--){
1126                 int type= s->input_picture[i]->pict_type;
1127                 if(type && type != FF_B_TYPE)
1128                     b_frames= i;
1129             }
1130             if(s->input_picture[b_frames]->pict_type == FF_B_TYPE && b_frames == s->max_b_frames){
1131                 av_log(s->avctx, AV_LOG_ERROR, "warning, too many b frames in a row\n");
1132             }
1133
1134             if(s->picture_in_gop_number + b_frames >= s->gop_size){
1135               if((s->flags2 & CODEC_FLAG2_STRICT_GOP) && s->gop_size > s->picture_in_gop_number){
1136                     b_frames= s->gop_size - s->picture_in_gop_number - 1;
1137               }else{
1138                 if(s->flags & CODEC_FLAG_CLOSED_GOP)
1139                     b_frames=0;
1140                 s->input_picture[b_frames]->pict_type= FF_I_TYPE;
1141               }
1142             }
1143
1144             if(   (s->flags & CODEC_FLAG_CLOSED_GOP)
1145                && b_frames
1146                && s->input_picture[b_frames]->pict_type== FF_I_TYPE)
1147                 b_frames--;
1148
1149             s->reordered_input_picture[0]= s->input_picture[b_frames];
1150             if(s->reordered_input_picture[0]->pict_type != FF_I_TYPE)
1151                 s->reordered_input_picture[0]->pict_type= FF_P_TYPE;
1152             s->reordered_input_picture[0]->coded_picture_number= s->coded_picture_number++;
1153             for(i=0; i<b_frames; i++){
1154                 s->reordered_input_picture[i+1]= s->input_picture[i];
1155                 s->reordered_input_picture[i+1]->pict_type= FF_B_TYPE;
1156                 s->reordered_input_picture[i+1]->coded_picture_number= s->coded_picture_number++;
1157             }
1158         }
1159     }
1160 no_output_pic:
1161     if(s->reordered_input_picture[0]){
1162         s->reordered_input_picture[0]->reference= s->reordered_input_picture[0]->pict_type!=FF_B_TYPE ? 3 : 0;
1163
1164         ff_copy_picture(&s->new_picture, s->reordered_input_picture[0]);
1165
1166         if(s->reordered_input_picture[0]->type == FF_BUFFER_TYPE_SHARED || s->avctx->rc_buffer_size){
1167             // input is a shared pix, so we can't modifiy it -> alloc a new one & ensure that the shared one is reuseable
1168
1169             int i= ff_find_unused_picture(s, 0);
1170             Picture *pic= &s->picture[i];
1171
1172             pic->reference              = s->reordered_input_picture[0]->reference;
1173             alloc_picture(s, pic, 0);
1174
1175             /* mark us unused / free shared pic */
1176             if(s->reordered_input_picture[0]->type == FF_BUFFER_TYPE_INTERNAL)
1177                 s->avctx->release_buffer(s->avctx, (AVFrame*)s->reordered_input_picture[0]);
1178             for(i=0; i<4; i++)
1179                 s->reordered_input_picture[0]->data[i]= NULL;
1180             s->reordered_input_picture[0]->type= 0;
1181
1182             copy_picture_attributes(s, (AVFrame*)pic, (AVFrame*)s->reordered_input_picture[0]);
1183
1184             s->current_picture_ptr= pic;
1185         }else{
1186             // input is not a shared pix -> reuse buffer for current_pix
1187
1188             assert(   s->reordered_input_picture[0]->type==FF_BUFFER_TYPE_USER
1189                    || s->reordered_input_picture[0]->type==FF_BUFFER_TYPE_INTERNAL);
1190
1191             s->current_picture_ptr= s->reordered_input_picture[0];
1192             for(i=0; i<4; i++){
1193                 s->new_picture.data[i]+= INPLACE_OFFSET;
1194             }
1195         }
1196         ff_copy_picture(&s->current_picture, s->current_picture_ptr);
1197
1198         s->picture_number= s->new_picture.display_picture_number;
1199 //printf("dpn:%d\n", s->picture_number);
1200     }else{
1201        memset(&s->new_picture, 0, sizeof(Picture));
1202     }
1203 }
1204
1205 int MPV_encode_picture(AVCodecContext *avctx,
1206                        unsigned char *buf, int buf_size, void *data)
1207 {
1208     MpegEncContext *s = avctx->priv_data;
1209     AVFrame *pic_arg = data;
1210     int i, stuffing_count;
1211
1212     for(i=0; i<avctx->thread_count; i++){
1213         int start_y= s->thread_context[i]->start_mb_y;
1214         int   end_y= s->thread_context[i]->  end_mb_y;
1215         int h= s->mb_height;
1216         uint8_t *start= buf + (size_t)(((int64_t) buf_size)*start_y/h);
1217         uint8_t *end  = buf + (size_t)(((int64_t) buf_size)*  end_y/h);
1218
1219         init_put_bits(&s->thread_context[i]->pb, start, end - start);
1220     }
1221
1222     s->picture_in_gop_number++;
1223
1224     if(load_input_picture(s, pic_arg) < 0)
1225         return -1;
1226
1227     select_input_picture(s);
1228
1229     /* output? */
1230     if(s->new_picture.data[0]){
1231         s->pict_type= s->new_picture.pict_type;
1232 //emms_c();
1233 //printf("qs:%f %f %d\n", s->new_picture.quality, s->current_picture.quality, s->qscale);
1234         MPV_frame_start(s, avctx);
1235 vbv_retry:
1236         if (encode_picture(s, s->picture_number) < 0)
1237             return -1;
1238
1239         avctx->real_pict_num  = s->picture_number;
1240         avctx->header_bits = s->header_bits;
1241         avctx->mv_bits     = s->mv_bits;
1242         avctx->misc_bits   = s->misc_bits;
1243         avctx->i_tex_bits  = s->i_tex_bits;
1244         avctx->p_tex_bits  = s->p_tex_bits;
1245         avctx->i_count     = s->i_count;
1246         avctx->p_count     = s->mb_num - s->i_count - s->skip_count; //FIXME f/b_count in avctx
1247         avctx->skip_count  = s->skip_count;
1248
1249         MPV_frame_end(s);
1250
1251         if (ENABLE_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1252             ff_mjpeg_encode_picture_trailer(s);
1253
1254         if(avctx->rc_buffer_size){
1255             RateControlContext *rcc= &s->rc_context;
1256             int max_size= rcc->buffer_index/3;
1257
1258             if(put_bits_count(&s->pb) > max_size && s->lambda < s->avctx->lmax){
1259                 s->next_lambda= FFMAX(s->lambda+1, s->lambda*(s->qscale+1) / s->qscale);
1260                 if(s->adaptive_quant){
1261                     int i;
1262                     for(i=0; i<s->mb_height*s->mb_stride; i++)
1263                         s->lambda_table[i]= FFMAX(s->lambda_table[i]+1, s->lambda_table[i]*(s->qscale+1) / s->qscale);
1264                 }
1265                 s->mb_skipped = 0;        //done in MPV_frame_start()
1266                 if(s->pict_type==FF_P_TYPE){ //done in encode_picture() so we must undo it
1267                     if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
1268                         s->no_rounding ^= 1;
1269                 }
1270                 if(s->pict_type!=FF_B_TYPE){
1271                     s->time_base= s->last_time_base;
1272                     s->last_non_b_time= s->time - s->pp_time;
1273                 }
1274 //                av_log(NULL, AV_LOG_ERROR, "R:%d ", s->next_lambda);
1275                 for(i=0; i<avctx->thread_count; i++){
1276                     PutBitContext *pb= &s->thread_context[i]->pb;
1277                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1278                 }
1279                 goto vbv_retry;
1280             }
1281
1282             assert(s->avctx->rc_max_rate);
1283         }
1284
1285         if(s->flags&CODEC_FLAG_PASS1)
1286             ff_write_pass1_stats(s);
1287
1288         for(i=0; i<4; i++){
1289             s->current_picture_ptr->error[i]= s->current_picture.error[i];
1290             avctx->error[i] += s->current_picture_ptr->error[i];
1291         }
1292
1293         if(s->flags&CODEC_FLAG_PASS1)
1294             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits + avctx->i_tex_bits + avctx->p_tex_bits == put_bits_count(&s->pb));
1295         flush_put_bits(&s->pb);
1296         s->frame_bits  = put_bits_count(&s->pb);
1297
1298         stuffing_count= ff_vbv_update(s, s->frame_bits);
1299         if(stuffing_count){
1300             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < stuffing_count + 50){
1301                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1302                 return -1;
1303             }
1304
1305             switch(s->codec_id){
1306             case CODEC_ID_MPEG1VIDEO:
1307             case CODEC_ID_MPEG2VIDEO:
1308                 while(stuffing_count--){
1309                     put_bits(&s->pb, 8, 0);
1310                 }
1311             break;
1312             case CODEC_ID_MPEG4:
1313                 put_bits(&s->pb, 16, 0);
1314                 put_bits(&s->pb, 16, 0x1C3);
1315                 stuffing_count -= 4;
1316                 while(stuffing_count--){
1317                     put_bits(&s->pb, 8, 0xFF);
1318                 }
1319             break;
1320             default:
1321                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1322             }
1323             flush_put_bits(&s->pb);
1324             s->frame_bits  = put_bits_count(&s->pb);
1325         }
1326
1327         /* update mpeg1/2 vbv_delay for CBR */
1328         if(s->avctx->rc_max_rate && s->avctx->rc_min_rate == s->avctx->rc_max_rate && s->out_format == FMT_MPEG1
1329            && 90000LL * (avctx->rc_buffer_size-1) <= s->avctx->rc_max_rate*0xFFFFLL){
1330             int vbv_delay;
1331
1332             assert(s->repeat_first_field==0);
1333
1334             vbv_delay= lrintf(90000 * s->rc_context.buffer_index / s->avctx->rc_max_rate);
1335             assert(vbv_delay < 0xFFFF);
1336
1337             s->vbv_delay_ptr[0] &= 0xF8;
1338             s->vbv_delay_ptr[0] |= vbv_delay>>13;
1339             s->vbv_delay_ptr[1]  = vbv_delay>>5;
1340             s->vbv_delay_ptr[2] &= 0x07;
1341             s->vbv_delay_ptr[2] |= vbv_delay<<3;
1342         }
1343         s->total_bits += s->frame_bits;
1344         avctx->frame_bits  = s->frame_bits;
1345     }else{
1346         assert((pbBufPtr(&s->pb) == s->pb.buf));
1347         s->frame_bits=0;
1348     }
1349     assert((s->frame_bits&7)==0);
1350
1351     return s->frame_bits/8;
1352 }
1353
1354 static inline void dct_single_coeff_elimination(MpegEncContext *s, int n, int threshold)
1355 {
1356     static const char tab[64]=
1357         {3,2,2,1,1,1,1,1,
1358          1,1,1,1,1,1,1,1,
1359          1,1,1,1,1,1,1,1,
1360          0,0,0,0,0,0,0,0,
1361          0,0,0,0,0,0,0,0,
1362          0,0,0,0,0,0,0,0,
1363          0,0,0,0,0,0,0,0,
1364          0,0,0,0,0,0,0,0};
1365     int score=0;
1366     int run=0;
1367     int i;
1368     DCTELEM *block= s->block[n];
1369     const int last_index= s->block_last_index[n];
1370     int skip_dc;
1371
1372     if(threshold<0){
1373         skip_dc=0;
1374         threshold= -threshold;
1375     }else
1376         skip_dc=1;
1377
1378     /* Are all we could set to zero already zero? */
1379     if(last_index<=skip_dc - 1) return;
1380
1381     for(i=0; i<=last_index; i++){
1382         const int j = s->intra_scantable.permutated[i];
1383         const int level = FFABS(block[j]);
1384         if(level==1){
1385             if(skip_dc && i==0) continue;
1386             score+= tab[run];
1387             run=0;
1388         }else if(level>1){
1389             return;
1390         }else{
1391             run++;
1392         }
1393     }
1394     if(score >= threshold) return;
1395     for(i=skip_dc; i<=last_index; i++){
1396         const int j = s->intra_scantable.permutated[i];
1397         block[j]=0;
1398     }
1399     if(block[0]) s->block_last_index[n]= 0;
1400     else         s->block_last_index[n]= -1;
1401 }
1402
1403 static inline void clip_coeffs(MpegEncContext *s, DCTELEM *block, int last_index)
1404 {
1405     int i;
1406     const int maxlevel= s->max_qcoeff;
1407     const int minlevel= s->min_qcoeff;
1408     int overflow=0;
1409
1410     if(s->mb_intra){
1411         i=1; //skip clipping of intra dc
1412     }else
1413         i=0;
1414
1415     for(;i<=last_index; i++){
1416         const int j= s->intra_scantable.permutated[i];
1417         int level = block[j];
1418
1419         if     (level>maxlevel){
1420             level=maxlevel;
1421             overflow++;
1422         }else if(level<minlevel){
1423             level=minlevel;
1424             overflow++;
1425         }
1426
1427         block[j]= level;
1428     }
1429
1430     if(overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1431         av_log(s->avctx, AV_LOG_INFO, "warning, clipping %d dct coefficients to %d..%d\n", overflow, minlevel, maxlevel);
1432 }
1433
1434 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride){
1435     int x, y;
1436 //FIXME optimize
1437     for(y=0; y<8; y++){
1438         for(x=0; x<8; x++){
1439             int x2, y2;
1440             int sum=0;
1441             int sqr=0;
1442             int count=0;
1443
1444             for(y2= FFMAX(y-1, 0); y2 < FFMIN(8, y+2); y2++){
1445                 for(x2= FFMAX(x-1, 0); x2 < FFMIN(8, x+2); x2++){
1446                     int v= ptr[x2 + y2*stride];
1447                     sum += v;
1448                     sqr += v*v;
1449                     count++;
1450                 }
1451             }
1452             weight[x + 8*y]= (36*ff_sqrt(count*sqr - sum*sum)) / count;
1453         }
1454     }
1455 }
1456
1457 static av_always_inline void encode_mb_internal(MpegEncContext *s, int motion_x, int motion_y, int mb_block_height, int mb_block_count)
1458 {
1459     int16_t weight[8][64];
1460     DCTELEM orig[8][64];
1461     const int mb_x= s->mb_x;
1462     const int mb_y= s->mb_y;
1463     int i;
1464     int skip_dct[8];
1465     int dct_offset   = s->linesize*8; //default for progressive frames
1466     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1467     int wrap_y, wrap_c;
1468
1469     for(i=0; i<mb_block_count; i++) skip_dct[i]=s->skipdct;
1470
1471     if(s->adaptive_quant){
1472         const int last_qp= s->qscale;
1473         const int mb_xy= mb_x + mb_y*s->mb_stride;
1474
1475         s->lambda= s->lambda_table[mb_xy];
1476         update_qscale(s);
1477
1478         if(!(s->flags&CODEC_FLAG_QP_RD)){
1479             s->qscale= s->current_picture_ptr->qscale_table[mb_xy];
1480             s->dquant= s->qscale - last_qp;
1481
1482             if(s->out_format==FMT_H263){
1483                 s->dquant= av_clip(s->dquant, -2, 2);
1484
1485                 if(s->codec_id==CODEC_ID_MPEG4){
1486                     if(!s->mb_intra){
1487                         if(s->pict_type == FF_B_TYPE){
1488                             if(s->dquant&1 || s->mv_dir&MV_DIRECT)
1489                                 s->dquant= 0;
1490                         }
1491                         if(s->mv_type==MV_TYPE_8X8)
1492                             s->dquant=0;
1493                     }
1494                 }
1495             }
1496         }
1497         ff_set_qscale(s, last_qp + s->dquant);
1498     }else if(s->flags&CODEC_FLAG_QP_RD)
1499         ff_set_qscale(s, s->qscale + s->dquant);
1500
1501     wrap_y = s->linesize;
1502     wrap_c = s->uvlinesize;
1503     ptr_y = s->new_picture.data[0] + (mb_y * 16 * wrap_y) + mb_x * 16;
1504     ptr_cb = s->new_picture.data[1] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1505     ptr_cr = s->new_picture.data[2] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1506
1507     if(mb_x*16+16 > s->width || mb_y*16+16 > s->height){
1508         uint8_t *ebuf= s->edge_emu_buffer + 32;
1509         ff_emulated_edge_mc(ebuf            , ptr_y , wrap_y,16,16,mb_x*16,mb_y*16, s->width   , s->height);
1510         ptr_y= ebuf;
1511         ff_emulated_edge_mc(ebuf+18*wrap_y  , ptr_cb, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1512         ptr_cb= ebuf+18*wrap_y;
1513         ff_emulated_edge_mc(ebuf+18*wrap_y+8, ptr_cr, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1514         ptr_cr= ebuf+18*wrap_y+8;
1515     }
1516
1517     if (s->mb_intra) {
1518         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1519             int progressive_score, interlaced_score;
1520
1521             s->interlaced_dct=0;
1522             progressive_score= s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y, 8)
1523                               +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y*8, NULL, wrap_y, 8) - 400;
1524
1525             if(progressive_score > 0){
1526                 interlaced_score = s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y*2, 8)
1527                                   +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y  , NULL, wrap_y*2, 8);
1528                 if(progressive_score > interlaced_score){
1529                     s->interlaced_dct=1;
1530
1531                     dct_offset= wrap_y;
1532                     wrap_y<<=1;
1533                     if (s->chroma_format == CHROMA_422)
1534                         wrap_c<<=1;
1535                 }
1536             }
1537         }
1538
1539         s->dsp.get_pixels(s->block[0], ptr_y                 , wrap_y);
1540         s->dsp.get_pixels(s->block[1], ptr_y              + 8, wrap_y);
1541         s->dsp.get_pixels(s->block[2], ptr_y + dct_offset    , wrap_y);
1542         s->dsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
1543
1544         if(s->flags&CODEC_FLAG_GRAY){
1545             skip_dct[4]= 1;
1546             skip_dct[5]= 1;
1547         }else{
1548             s->dsp.get_pixels(s->block[4], ptr_cb, wrap_c);
1549             s->dsp.get_pixels(s->block[5], ptr_cr, wrap_c);
1550             if(!s->chroma_y_shift){ /* 422 */
1551                 s->dsp.get_pixels(s->block[6], ptr_cb + (dct_offset>>1), wrap_c);
1552                 s->dsp.get_pixels(s->block[7], ptr_cr + (dct_offset>>1), wrap_c);
1553             }
1554         }
1555     }else{
1556         op_pixels_func (*op_pix)[4];
1557         qpel_mc_func (*op_qpix)[16];
1558         uint8_t *dest_y, *dest_cb, *dest_cr;
1559
1560         dest_y  = s->dest[0];
1561         dest_cb = s->dest[1];
1562         dest_cr = s->dest[2];
1563
1564         if ((!s->no_rounding) || s->pict_type==FF_B_TYPE){
1565             op_pix = s->dsp.put_pixels_tab;
1566             op_qpix= s->dsp.put_qpel_pixels_tab;
1567         }else{
1568             op_pix = s->dsp.put_no_rnd_pixels_tab;
1569             op_qpix= s->dsp.put_no_rnd_qpel_pixels_tab;
1570         }
1571
1572         if (s->mv_dir & MV_DIR_FORWARD) {
1573             MPV_motion(s, dest_y, dest_cb, dest_cr, 0, s->last_picture.data, op_pix, op_qpix);
1574             op_pix = s->dsp.avg_pixels_tab;
1575             op_qpix= s->dsp.avg_qpel_pixels_tab;
1576         }
1577         if (s->mv_dir & MV_DIR_BACKWARD) {
1578             MPV_motion(s, dest_y, dest_cb, dest_cr, 1, s->next_picture.data, op_pix, op_qpix);
1579         }
1580
1581         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1582             int progressive_score, interlaced_score;
1583
1584             s->interlaced_dct=0;
1585             progressive_score= s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y, 8)
1586                               +s->dsp.ildct_cmp[0](s, dest_y + wrap_y*8, ptr_y + wrap_y*8, wrap_y, 8) - 400;
1587
1588             if(s->avctx->ildct_cmp == FF_CMP_VSSE) progressive_score -= 400;
1589
1590             if(progressive_score>0){
1591                 interlaced_score = s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y*2, 8)
1592                                   +s->dsp.ildct_cmp[0](s, dest_y + wrap_y  , ptr_y + wrap_y  , wrap_y*2, 8);
1593
1594                 if(progressive_score > interlaced_score){
1595                     s->interlaced_dct=1;
1596
1597                     dct_offset= wrap_y;
1598                     wrap_y<<=1;
1599                     if (s->chroma_format == CHROMA_422)
1600                         wrap_c<<=1;
1601                 }
1602             }
1603         }
1604
1605         s->dsp.diff_pixels(s->block[0], ptr_y                 , dest_y                 , wrap_y);
1606         s->dsp.diff_pixels(s->block[1], ptr_y              + 8, dest_y              + 8, wrap_y);
1607         s->dsp.diff_pixels(s->block[2], ptr_y + dct_offset    , dest_y + dct_offset    , wrap_y);
1608         s->dsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8, dest_y + dct_offset + 8, wrap_y);
1609
1610         if(s->flags&CODEC_FLAG_GRAY){
1611             skip_dct[4]= 1;
1612             skip_dct[5]= 1;
1613         }else{
1614             s->dsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
1615             s->dsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
1616             if(!s->chroma_y_shift){ /* 422 */
1617                 s->dsp.diff_pixels(s->block[6], ptr_cb + (dct_offset>>1), dest_cb + (dct_offset>>1), wrap_c);
1618                 s->dsp.diff_pixels(s->block[7], ptr_cr + (dct_offset>>1), dest_cr + (dct_offset>>1), wrap_c);
1619             }
1620         }
1621         /* pre quantization */
1622         if(s->current_picture.mc_mb_var[s->mb_stride*mb_y+ mb_x]<2*s->qscale*s->qscale){
1623             //FIXME optimize
1624             if(s->dsp.sad[1](NULL, ptr_y               , dest_y               , wrap_y, 8) < 20*s->qscale) skip_dct[0]= 1;
1625             if(s->dsp.sad[1](NULL, ptr_y            + 8, dest_y            + 8, wrap_y, 8) < 20*s->qscale) skip_dct[1]= 1;
1626             if(s->dsp.sad[1](NULL, ptr_y +dct_offset   , dest_y +dct_offset   , wrap_y, 8) < 20*s->qscale) skip_dct[2]= 1;
1627             if(s->dsp.sad[1](NULL, ptr_y +dct_offset+ 8, dest_y +dct_offset+ 8, wrap_y, 8) < 20*s->qscale) skip_dct[3]= 1;
1628             if(s->dsp.sad[1](NULL, ptr_cb              , dest_cb              , wrap_c, 8) < 20*s->qscale) skip_dct[4]= 1;
1629             if(s->dsp.sad[1](NULL, ptr_cr              , dest_cr              , wrap_c, 8) < 20*s->qscale) skip_dct[5]= 1;
1630             if(!s->chroma_y_shift){ /* 422 */
1631                 if(s->dsp.sad[1](NULL, ptr_cb +(dct_offset>>1), dest_cb +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[6]= 1;
1632                 if(s->dsp.sad[1](NULL, ptr_cr +(dct_offset>>1), dest_cr +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[7]= 1;
1633             }
1634         }
1635     }
1636
1637     if(s->avctx->quantizer_noise_shaping){
1638         if(!skip_dct[0]) get_visual_weight(weight[0], ptr_y                 , wrap_y);
1639         if(!skip_dct[1]) get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
1640         if(!skip_dct[2]) get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
1641         if(!skip_dct[3]) get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
1642         if(!skip_dct[4]) get_visual_weight(weight[4], ptr_cb                , wrap_c);
1643         if(!skip_dct[5]) get_visual_weight(weight[5], ptr_cr                , wrap_c);
1644         if(!s->chroma_y_shift){ /* 422 */
1645             if(!skip_dct[6]) get_visual_weight(weight[6], ptr_cb + (dct_offset>>1), wrap_c);
1646             if(!skip_dct[7]) get_visual_weight(weight[7], ptr_cr + (dct_offset>>1), wrap_c);
1647         }
1648         memcpy(orig[0], s->block[0], sizeof(DCTELEM)*64*mb_block_count);
1649     }
1650
1651     /* DCT & quantize */
1652     assert(s->out_format!=FMT_MJPEG || s->qscale==8);
1653     {
1654         for(i=0;i<mb_block_count;i++) {
1655             if(!skip_dct[i]){
1656                 int overflow;
1657                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
1658             // FIXME we could decide to change to quantizer instead of clipping
1659             // JS: I don't think that would be a good idea it could lower quality instead
1660             //     of improve it. Just INTRADC clipping deserves changes in quantizer
1661                 if (overflow) clip_coeffs(s, s->block[i], s->block_last_index[i]);
1662             }else
1663                 s->block_last_index[i]= -1;
1664         }
1665         if(s->avctx->quantizer_noise_shaping){
1666             for(i=0;i<mb_block_count;i++) {
1667                 if(!skip_dct[i]){
1668                     s->block_last_index[i] = dct_quantize_refine(s, s->block[i], weight[i], orig[i], i, s->qscale);
1669                 }
1670             }
1671         }
1672
1673         if(s->luma_elim_threshold && !s->mb_intra)
1674             for(i=0; i<4; i++)
1675                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
1676         if(s->chroma_elim_threshold && !s->mb_intra)
1677             for(i=4; i<mb_block_count; i++)
1678                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
1679
1680         if(s->flags & CODEC_FLAG_CBP_RD){
1681             for(i=0;i<mb_block_count;i++) {
1682                 if(s->block_last_index[i] == -1)
1683                     s->coded_score[i]= INT_MAX/256;
1684             }
1685         }
1686     }
1687
1688     if((s->flags&CODEC_FLAG_GRAY) && s->mb_intra){
1689         s->block_last_index[4]=
1690         s->block_last_index[5]= 0;
1691         s->block[4][0]=
1692         s->block[5][0]= (1024 + s->c_dc_scale/2)/ s->c_dc_scale;
1693     }
1694
1695     //non c quantize code returns incorrect block_last_index FIXME
1696     if(s->alternate_scan && s->dct_quantize != dct_quantize_c){
1697         for(i=0; i<mb_block_count; i++){
1698             int j;
1699             if(s->block_last_index[i]>0){
1700                 for(j=63; j>0; j--){
1701                     if(s->block[i][ s->intra_scantable.permutated[j] ]) break;
1702                 }
1703                 s->block_last_index[i]= j;
1704             }
1705         }
1706     }
1707
1708     /* huffman encode */
1709     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
1710     case CODEC_ID_MPEG1VIDEO:
1711     case CODEC_ID_MPEG2VIDEO:
1712         if (ENABLE_MPEG1VIDEO_ENCODER || ENABLE_MPEG2VIDEO_ENCODER)
1713             mpeg1_encode_mb(s, s->block, motion_x, motion_y);
1714         break;
1715     case CODEC_ID_MPEG4:
1716         if (ENABLE_MPEG4_ENCODER)
1717             mpeg4_encode_mb(s, s->block, motion_x, motion_y);
1718         break;
1719     case CODEC_ID_MSMPEG4V2:
1720     case CODEC_ID_MSMPEG4V3:
1721     case CODEC_ID_WMV1:
1722         if (ENABLE_MSMPEG4_ENCODER)
1723             msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
1724         break;
1725     case CODEC_ID_WMV2:
1726         if (ENABLE_WMV2_ENCODER)
1727             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
1728         break;
1729     case CODEC_ID_H261:
1730         if (ENABLE_H261_ENCODER)
1731             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
1732         break;
1733     case CODEC_ID_H263:
1734     case CODEC_ID_H263P:
1735     case CODEC_ID_FLV1:
1736     case CODEC_ID_RV10:
1737     case CODEC_ID_RV20:
1738         if (ENABLE_H263_ENCODER || ENABLE_H263P_ENCODER ||
1739             ENABLE_FLV_ENCODER  || ENABLE_RV10_ENCODER  || ENABLE_RV20_ENCODER)
1740             h263_encode_mb(s, s->block, motion_x, motion_y);
1741         break;
1742     case CODEC_ID_MJPEG:
1743         if (ENABLE_MJPEG_ENCODER)
1744             ff_mjpeg_encode_mb(s, s->block);
1745         break;
1746     default:
1747         assert(0);
1748     }
1749 }
1750
1751 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
1752 {
1753     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
1754     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
1755 }
1756
1757 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
1758     int i;
1759
1760     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster then a loop?
1761
1762     /* mpeg1 */
1763     d->mb_skip_run= s->mb_skip_run;
1764     for(i=0; i<3; i++)
1765         d->last_dc[i]= s->last_dc[i];
1766
1767     /* statistics */
1768     d->mv_bits= s->mv_bits;
1769     d->i_tex_bits= s->i_tex_bits;
1770     d->p_tex_bits= s->p_tex_bits;
1771     d->i_count= s->i_count;
1772     d->f_count= s->f_count;
1773     d->b_count= s->b_count;
1774     d->skip_count= s->skip_count;
1775     d->misc_bits= s->misc_bits;
1776     d->last_bits= 0;
1777
1778     d->mb_skipped= 0;
1779     d->qscale= s->qscale;
1780     d->dquant= s->dquant;
1781
1782     d->esc3_level_length= s->esc3_level_length;
1783 }
1784
1785 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
1786     int i;
1787
1788     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
1789     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster then a loop?
1790
1791     /* mpeg1 */
1792     d->mb_skip_run= s->mb_skip_run;
1793     for(i=0; i<3; i++)
1794         d->last_dc[i]= s->last_dc[i];
1795
1796     /* statistics */
1797     d->mv_bits= s->mv_bits;
1798     d->i_tex_bits= s->i_tex_bits;
1799     d->p_tex_bits= s->p_tex_bits;
1800     d->i_count= s->i_count;
1801     d->f_count= s->f_count;
1802     d->b_count= s->b_count;
1803     d->skip_count= s->skip_count;
1804     d->misc_bits= s->misc_bits;
1805
1806     d->mb_intra= s->mb_intra;
1807     d->mb_skipped= s->mb_skipped;
1808     d->mv_type= s->mv_type;
1809     d->mv_dir= s->mv_dir;
1810     d->pb= s->pb;
1811     if(s->data_partitioning){
1812         d->pb2= s->pb2;
1813         d->tex_pb= s->tex_pb;
1814     }
1815     d->block= s->block;
1816     for(i=0; i<8; i++)
1817         d->block_last_index[i]= s->block_last_index[i];
1818     d->interlaced_dct= s->interlaced_dct;
1819     d->qscale= s->qscale;
1820
1821     d->esc3_level_length= s->esc3_level_length;
1822 }
1823
1824 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
1825                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
1826                            int *dmin, int *next_block, int motion_x, int motion_y)
1827 {
1828     int score;
1829     uint8_t *dest_backup[3];
1830
1831     copy_context_before_encode(s, backup, type);
1832
1833     s->block= s->blocks[*next_block];
1834     s->pb= pb[*next_block];
1835     if(s->data_partitioning){
1836         s->pb2   = pb2   [*next_block];
1837         s->tex_pb= tex_pb[*next_block];
1838     }
1839
1840     if(*next_block){
1841         memcpy(dest_backup, s->dest, sizeof(s->dest));
1842         s->dest[0] = s->rd_scratchpad;
1843         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
1844         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
1845         assert(s->linesize >= 32); //FIXME
1846     }
1847
1848     encode_mb(s, motion_x, motion_y);
1849
1850     score= put_bits_count(&s->pb);
1851     if(s->data_partitioning){
1852         score+= put_bits_count(&s->pb2);
1853         score+= put_bits_count(&s->tex_pb);
1854     }
1855
1856     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
1857         MPV_decode_mb(s, s->block);
1858
1859         score *= s->lambda2;
1860         score += sse_mb(s) << FF_LAMBDA_SHIFT;
1861     }
1862
1863     if(*next_block){
1864         memcpy(s->dest, dest_backup, sizeof(s->dest));
1865     }
1866
1867     if(score<*dmin){
1868         *dmin= score;
1869         *next_block^=1;
1870
1871         copy_context_after_encode(best, s, type);
1872     }
1873 }
1874
1875 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
1876     uint32_t *sq = ff_squareTbl + 256;
1877     int acc=0;
1878     int x,y;
1879
1880     if(w==16 && h==16)
1881         return s->dsp.sse[0](NULL, src1, src2, stride, 16);
1882     else if(w==8 && h==8)
1883         return s->dsp.sse[1](NULL, src1, src2, stride, 8);
1884
1885     for(y=0; y<h; y++){
1886         for(x=0; x<w; x++){
1887             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
1888         }
1889     }
1890
1891     assert(acc>=0);
1892
1893     return acc;
1894 }
1895
1896 static int sse_mb(MpegEncContext *s){
1897     int w= 16;
1898     int h= 16;
1899
1900     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
1901     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
1902
1903     if(w==16 && h==16)
1904       if(s->avctx->mb_cmp == FF_CMP_NSSE){
1905         return  s->dsp.nsse[0](s, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
1906                +s->dsp.nsse[1](s, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
1907                +s->dsp.nsse[1](s, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
1908       }else{
1909         return  s->dsp.sse[0](NULL, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
1910                +s->dsp.sse[1](NULL, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
1911                +s->dsp.sse[1](NULL, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
1912       }
1913     else
1914         return  sse(s, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
1915                +sse(s, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
1916                +sse(s, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
1917 }
1918
1919 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
1920     MpegEncContext *s= *(void**)arg;
1921
1922
1923     s->me.pre_pass=1;
1924     s->me.dia_size= s->avctx->pre_dia_size;
1925     s->first_slice_line=1;
1926     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
1927         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
1928             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
1929         }
1930         s->first_slice_line=0;
1931     }
1932
1933     s->me.pre_pass=0;
1934
1935     return 0;
1936 }
1937
1938 static int estimate_motion_thread(AVCodecContext *c, void *arg){
1939     MpegEncContext *s= *(void**)arg;
1940
1941     ff_check_alignment();
1942
1943     s->me.dia_size= s->avctx->dia_size;
1944     s->first_slice_line=1;
1945     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
1946         s->mb_x=0; //for block init below
1947         ff_init_block_index(s);
1948         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
1949             s->block_index[0]+=2;
1950             s->block_index[1]+=2;
1951             s->block_index[2]+=2;
1952             s->block_index[3]+=2;
1953
1954             /* compute motion vector & mb_type and store in context */
1955             if(s->pict_type==FF_B_TYPE)
1956                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
1957             else
1958                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
1959         }
1960         s->first_slice_line=0;
1961     }
1962     return 0;
1963 }
1964
1965 static int mb_var_thread(AVCodecContext *c, void *arg){
1966     MpegEncContext *s= *(void**)arg;
1967     int mb_x, mb_y;
1968
1969     ff_check_alignment();
1970
1971     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
1972         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
1973             int xx = mb_x * 16;
1974             int yy = mb_y * 16;
1975             uint8_t *pix = s->new_picture.data[0] + (yy * s->linesize) + xx;
1976             int varc;
1977             int sum = s->dsp.pix_sum(pix, s->linesize);
1978
1979             varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)(sum*sum))>>8) + 500 + 128)>>8;
1980
1981             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
1982             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
1983             s->me.mb_var_sum_temp    += varc;
1984         }
1985     }
1986     return 0;
1987 }
1988
1989 static void write_slice_end(MpegEncContext *s){
1990     if(ENABLE_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4){
1991         if(s->partitioned_frame){
1992             ff_mpeg4_merge_partitions(s);
1993         }
1994
1995         ff_mpeg4_stuffing(&s->pb);
1996     }else if(ENABLE_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
1997         ff_mjpeg_encode_stuffing(&s->pb);
1998     }
1999
2000     align_put_bits(&s->pb);
2001     flush_put_bits(&s->pb);
2002
2003     if((s->flags&CODEC_FLAG_PASS1) && !s->partitioned_frame)
2004         s->misc_bits+= get_bits_diff(s);
2005 }
2006
2007 static int encode_thread(AVCodecContext *c, void *arg){
2008     MpegEncContext *s= *(void**)arg;
2009     int mb_x, mb_y, pdif = 0;
2010     int chr_h= 16>>s->chroma_y_shift;
2011     int i, j;
2012     MpegEncContext best_s, backup_s;
2013     uint8_t bit_buf[2][MAX_MB_BYTES];
2014     uint8_t bit_buf2[2][MAX_MB_BYTES];
2015     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2016     PutBitContext pb[2], pb2[2], tex_pb[2];
2017 //printf("%d->%d\n", s->resync_mb_y, s->end_mb_y);
2018
2019     ff_check_alignment();
2020
2021     for(i=0; i<2; i++){
2022         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2023         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2024         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2025     }
2026
2027     s->last_bits= put_bits_count(&s->pb);
2028     s->mv_bits=0;
2029     s->misc_bits=0;
2030     s->i_tex_bits=0;
2031     s->p_tex_bits=0;
2032     s->i_count=0;
2033     s->f_count=0;
2034     s->b_count=0;
2035     s->skip_count=0;
2036
2037     for(i=0; i<3; i++){
2038         /* init last dc values */
2039         /* note: quant matrix value (8) is implied here */
2040         s->last_dc[i] = 128 << s->intra_dc_precision;
2041
2042         s->current_picture.error[i] = 0;
2043     }
2044     s->mb_skip_run = 0;
2045     memset(s->last_mv, 0, sizeof(s->last_mv));
2046
2047     s->last_mv_dir = 0;
2048
2049     switch(s->codec_id){
2050     case CODEC_ID_H263:
2051     case CODEC_ID_H263P:
2052     case CODEC_ID_FLV1:
2053         if (ENABLE_H263_ENCODER || ENABLE_H263P_ENCODER || ENABLE_FLV_ENCODER)
2054             s->gob_index = ff_h263_get_gob_height(s);
2055         break;
2056     case CODEC_ID_MPEG4:
2057         if(ENABLE_MPEG4_ENCODER && s->partitioned_frame)
2058             ff_mpeg4_init_partitions(s);
2059         break;
2060     }
2061
2062     s->resync_mb_x=0;
2063     s->resync_mb_y=0;
2064     s->first_slice_line = 1;
2065     s->ptr_lastgob = s->pb.buf;
2066     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2067 //    printf("row %d at %X\n", s->mb_y, (int)s);
2068         s->mb_x=0;
2069         s->mb_y= mb_y;
2070
2071         ff_set_qscale(s, s->qscale);
2072         ff_init_block_index(s);
2073
2074         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2075             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2076             int mb_type= s->mb_type[xy];
2077 //            int d;
2078             int dmin= INT_MAX;
2079             int dir;
2080
2081             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2082                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2083                 return -1;
2084             }
2085             if(s->data_partitioning){
2086                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2087                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2088                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2089                     return -1;
2090                 }
2091             }
2092
2093             s->mb_x = mb_x;
2094             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2095             ff_update_block_index(s);
2096
2097             if(ENABLE_H261_ENCODER && s->codec_id == CODEC_ID_H261){
2098                 ff_h261_reorder_mb_index(s);
2099                 xy= s->mb_y*s->mb_stride + s->mb_x;
2100                 mb_type= s->mb_type[xy];
2101             }
2102
2103             /* write gob / video packet header  */
2104             if(s->rtp_mode){
2105                 int current_packet_size, is_gob_start;
2106
2107                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2108
2109                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2110
2111                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2112
2113                 switch(s->codec_id){
2114                 case CODEC_ID_H263:
2115                 case CODEC_ID_H263P:
2116                     if(!s->h263_slice_structured)
2117                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2118                     break;
2119                 case CODEC_ID_MPEG2VIDEO:
2120                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2121                 case CODEC_ID_MPEG1VIDEO:
2122                     if(s->mb_skip_run) is_gob_start=0;
2123                     break;
2124                 }
2125
2126                 if(is_gob_start){
2127                     if(s->start_mb_y != mb_y || mb_x!=0){
2128                         write_slice_end(s);
2129
2130                         if(ENABLE_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4 && s->partitioned_frame){
2131                             ff_mpeg4_init_partitions(s);
2132                         }
2133                     }
2134
2135                     assert((put_bits_count(&s->pb)&7) == 0);
2136                     current_packet_size= pbBufPtr(&s->pb) - s->ptr_lastgob;
2137
2138                     if(s->avctx->error_rate && s->resync_mb_x + s->resync_mb_y > 0){
2139                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2140                         int d= 100 / s->avctx->error_rate;
2141                         if(r % d == 0){
2142                             current_packet_size=0;
2143 #ifndef ALT_BITSTREAM_WRITER
2144                             s->pb.buf_ptr= s->ptr_lastgob;
2145 #endif
2146                             assert(pbBufPtr(&s->pb) == s->ptr_lastgob);
2147                         }
2148                     }
2149
2150                     if (s->avctx->rtp_callback){
2151                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2152                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2153                     }
2154
2155                     switch(s->codec_id){
2156                     case CODEC_ID_MPEG4:
2157                         if (ENABLE_MPEG4_ENCODER) {
2158                             ff_mpeg4_encode_video_packet_header(s);
2159                             ff_mpeg4_clean_buffers(s);
2160                         }
2161                     break;
2162                     case CODEC_ID_MPEG1VIDEO:
2163                     case CODEC_ID_MPEG2VIDEO:
2164                         if (ENABLE_MPEG1VIDEO_ENCODER || ENABLE_MPEG2VIDEO_ENCODER) {
2165                             ff_mpeg1_encode_slice_header(s);
2166                             ff_mpeg1_clean_buffers(s);
2167                         }
2168                     break;
2169                     case CODEC_ID_H263:
2170                     case CODEC_ID_H263P:
2171                         if (ENABLE_H263_ENCODER || ENABLE_H263P_ENCODER)
2172                             h263_encode_gob_header(s, mb_y);
2173                     break;
2174                     }
2175
2176                     if(s->flags&CODEC_FLAG_PASS1){
2177                         int bits= put_bits_count(&s->pb);
2178                         s->misc_bits+= bits - s->last_bits;
2179                         s->last_bits= bits;
2180                     }
2181
2182                     s->ptr_lastgob += current_packet_size;
2183                     s->first_slice_line=1;
2184                     s->resync_mb_x=mb_x;
2185                     s->resync_mb_y=mb_y;
2186                 }
2187             }
2188
2189             if(  (s->resync_mb_x   == s->mb_x)
2190                && s->resync_mb_y+1 == s->mb_y){
2191                 s->first_slice_line=0;
2192             }
2193
2194             s->mb_skipped=0;
2195             s->dquant=0; //only for QP_RD
2196
2197             if(mb_type & (mb_type-1) || (s->flags & CODEC_FLAG_QP_RD)){ // more than 1 MB type possible or CODEC_FLAG_QP_RD
2198                 int next_block=0;
2199                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2200
2201                 copy_context_before_encode(&backup_s, s, -1);
2202                 backup_s.pb= s->pb;
2203                 best_s.data_partitioning= s->data_partitioning;
2204                 best_s.partitioned_frame= s->partitioned_frame;
2205                 if(s->data_partitioning){
2206                     backup_s.pb2= s->pb2;
2207                     backup_s.tex_pb= s->tex_pb;
2208                 }
2209
2210                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2211                     s->mv_dir = MV_DIR_FORWARD;
2212                     s->mv_type = MV_TYPE_16X16;
2213                     s->mb_intra= 0;
2214                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2215                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2216                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2217                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2218                 }
2219                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2220                     s->mv_dir = MV_DIR_FORWARD;
2221                     s->mv_type = MV_TYPE_FIELD;
2222                     s->mb_intra= 0;
2223                     for(i=0; i<2; i++){
2224                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2225                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2226                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2227                     }
2228                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2229                                  &dmin, &next_block, 0, 0);
2230                 }
2231                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2232                     s->mv_dir = MV_DIR_FORWARD;
2233                     s->mv_type = MV_TYPE_16X16;
2234                     s->mb_intra= 0;
2235                     s->mv[0][0][0] = 0;
2236                     s->mv[0][0][1] = 0;
2237                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2238                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2239                 }
2240                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2241                     s->mv_dir = MV_DIR_FORWARD;
2242                     s->mv_type = MV_TYPE_8X8;
2243                     s->mb_intra= 0;
2244                     for(i=0; i<4; i++){
2245                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2246                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2247                     }
2248                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2249                                  &dmin, &next_block, 0, 0);
2250                 }
2251                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2252                     s->mv_dir = MV_DIR_FORWARD;
2253                     s->mv_type = MV_TYPE_16X16;
2254                     s->mb_intra= 0;
2255                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2256                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2257                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2258                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2259                 }
2260                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2261                     s->mv_dir = MV_DIR_BACKWARD;
2262                     s->mv_type = MV_TYPE_16X16;
2263                     s->mb_intra= 0;
2264                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2265                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2266                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2267                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2268                 }
2269                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2270                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2271                     s->mv_type = MV_TYPE_16X16;
2272                     s->mb_intra= 0;
2273                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2274                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2275                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2276                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2277                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2278                                  &dmin, &next_block, 0, 0);
2279                 }
2280                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2281                     s->mv_dir = MV_DIR_FORWARD;
2282                     s->mv_type = MV_TYPE_FIELD;
2283                     s->mb_intra= 0;
2284                     for(i=0; i<2; i++){
2285                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2286                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2287                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2288                     }
2289                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2290                                  &dmin, &next_block, 0, 0);
2291                 }
2292                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2293                     s->mv_dir = MV_DIR_BACKWARD;
2294                     s->mv_type = MV_TYPE_FIELD;
2295                     s->mb_intra= 0;
2296                     for(i=0; i<2; i++){
2297                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2298                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2299                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2300                     }
2301                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2302                                  &dmin, &next_block, 0, 0);
2303                 }
2304                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2305                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2306                     s->mv_type = MV_TYPE_FIELD;
2307                     s->mb_intra= 0;
2308                     for(dir=0; dir<2; dir++){
2309                         for(i=0; i<2; i++){
2310                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2311                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2312                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2313                         }
2314                     }
2315                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2316                                  &dmin, &next_block, 0, 0);
2317                 }
2318                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2319                     s->mv_dir = 0;
2320                     s->mv_type = MV_TYPE_16X16;
2321                     s->mb_intra= 1;
2322                     s->mv[0][0][0] = 0;
2323                     s->mv[0][0][1] = 0;
2324                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2325                                  &dmin, &next_block, 0, 0);
2326                     if(s->h263_pred || s->h263_aic){
2327                         if(best_s.mb_intra)
2328                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2329                         else
2330                             ff_clean_intra_table_entries(s); //old mode?
2331                     }
2332                 }
2333
2334                 if((s->flags & CODEC_FLAG_QP_RD) && dmin < INT_MAX){
2335                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2336                         const int last_qp= backup_s.qscale;
2337                         int qpi, qp, dc[6];
2338                         DCTELEM ac[6][16];
2339                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2340                         static const int dquant_tab[4]={-1,1,-2,2};
2341
2342                         assert(backup_s.dquant == 0);
2343
2344                         //FIXME intra
2345                         s->mv_dir= best_s.mv_dir;
2346                         s->mv_type = MV_TYPE_16X16;
2347                         s->mb_intra= best_s.mb_intra;
2348                         s->mv[0][0][0] = best_s.mv[0][0][0];
2349                         s->mv[0][0][1] = best_s.mv[0][0][1];
2350                         s->mv[1][0][0] = best_s.mv[1][0][0];
2351                         s->mv[1][0][1] = best_s.mv[1][0][1];
2352
2353                         qpi = s->pict_type == FF_B_TYPE ? 2 : 0;
2354                         for(; qpi<4; qpi++){
2355                             int dquant= dquant_tab[qpi];
2356                             qp= last_qp + dquant;
2357                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2358                                 continue;
2359                             backup_s.dquant= dquant;
2360                             if(s->mb_intra && s->dc_val[0]){
2361                                 for(i=0; i<6; i++){
2362                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2363                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(DCTELEM)*16);
2364                                 }
2365                             }
2366
2367                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2368                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2369                             if(best_s.qscale != qp){
2370                                 if(s->mb_intra && s->dc_val[0]){
2371                                     for(i=0; i<6; i++){
2372                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2373                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(DCTELEM)*16);
2374                                     }
2375                                 }
2376                             }
2377                         }
2378                     }
2379                 }
2380                 if(ENABLE_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2381                     int mx= s->b_direct_mv_table[xy][0];
2382                     int my= s->b_direct_mv_table[xy][1];
2383
2384                     backup_s.dquant = 0;
2385                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2386                     s->mb_intra= 0;
2387                     ff_mpeg4_set_direct_mv(s, mx, my);
2388                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2389                                  &dmin, &next_block, mx, my);
2390                 }
2391                 if(ENABLE_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2392                     backup_s.dquant = 0;
2393                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2394                     s->mb_intra= 0;
2395                     ff_mpeg4_set_direct_mv(s, 0, 0);
2396                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2397                                  &dmin, &next_block, 0, 0);
2398                 }
2399                 if(!best_s.mb_intra && s->flags2&CODEC_FLAG2_SKIP_RD){
2400                     int coded=0;
2401                     for(i=0; i<6; i++)
2402                         coded |= s->block_last_index[i];
2403                     if(coded){
2404                         int mx,my;
2405                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2406                         if(ENABLE_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
2407                             mx=my=0; //FIXME find the one we actually used
2408                             ff_mpeg4_set_direct_mv(s, mx, my);
2409                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2410                             mx= s->mv[1][0][0];
2411                             my= s->mv[1][0][1];
2412                         }else{
2413                             mx= s->mv[0][0][0];
2414                             my= s->mv[0][0][1];
2415                         }
2416
2417                         s->mv_dir= best_s.mv_dir;
2418                         s->mv_type = best_s.mv_type;
2419                         s->mb_intra= 0;
2420 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2421                         s->mv[0][0][1] = best_s.mv[0][0][1];
2422                         s->mv[1][0][0] = best_s.mv[1][0][0];
2423                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2424                         backup_s.dquant= 0;
2425                         s->skipdct=1;
2426                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2427                                         &dmin, &next_block, mx, my);
2428                         s->skipdct=0;
2429                     }
2430                 }
2431
2432                 s->current_picture.qscale_table[xy]= best_s.qscale;
2433
2434                 copy_context_after_encode(s, &best_s, -1);
2435
2436                 pb_bits_count= put_bits_count(&s->pb);
2437                 flush_put_bits(&s->pb);
2438                 ff_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2439                 s->pb= backup_s.pb;
2440
2441                 if(s->data_partitioning){
2442                     pb2_bits_count= put_bits_count(&s->pb2);
2443                     flush_put_bits(&s->pb2);
2444                     ff_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2445                     s->pb2= backup_s.pb2;
2446
2447                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2448                     flush_put_bits(&s->tex_pb);
2449                     ff_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2450                     s->tex_pb= backup_s.tex_pb;
2451                 }
2452                 s->last_bits= put_bits_count(&s->pb);
2453
2454                 if (ENABLE_ANY_H263_ENCODER &&
2455                     s->out_format == FMT_H263 && s->pict_type!=FF_B_TYPE)
2456                     ff_h263_update_motion_val(s);
2457
2458                 if(next_block==0){ //FIXME 16 vs linesize16
2459                     s->dsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
2460                     s->dsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2461                     s->dsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2462                 }
2463
2464                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
2465                     MPV_decode_mb(s, s->block);
2466             } else {
2467                 int motion_x = 0, motion_y = 0;
2468                 s->mv_type=MV_TYPE_16X16;
2469                 // only one MB-Type possible
2470
2471                 switch(mb_type){
2472                 case CANDIDATE_MB_TYPE_INTRA:
2473                     s->mv_dir = 0;
2474                     s->mb_intra= 1;
2475                     motion_x= s->mv[0][0][0] = 0;
2476                     motion_y= s->mv[0][0][1] = 0;
2477                     break;
2478                 case CANDIDATE_MB_TYPE_INTER:
2479                     s->mv_dir = MV_DIR_FORWARD;
2480                     s->mb_intra= 0;
2481                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
2482                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
2483                     break;
2484                 case CANDIDATE_MB_TYPE_INTER_I:
2485                     s->mv_dir = MV_DIR_FORWARD;
2486                     s->mv_type = MV_TYPE_FIELD;
2487                     s->mb_intra= 0;
2488                     for(i=0; i<2; i++){
2489                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2490                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2491                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2492                     }
2493                     break;
2494                 case CANDIDATE_MB_TYPE_INTER4V:
2495                     s->mv_dir = MV_DIR_FORWARD;
2496                     s->mv_type = MV_TYPE_8X8;
2497                     s->mb_intra= 0;
2498                     for(i=0; i<4; i++){
2499                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2500                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2501                     }
2502                     break;
2503                 case CANDIDATE_MB_TYPE_DIRECT:
2504                     if (ENABLE_MPEG4_ENCODER) {
2505                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2506                         s->mb_intra= 0;
2507                         motion_x=s->b_direct_mv_table[xy][0];
2508                         motion_y=s->b_direct_mv_table[xy][1];
2509                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
2510                     }
2511                     break;
2512                 case CANDIDATE_MB_TYPE_DIRECT0:
2513                     if (ENABLE_MPEG4_ENCODER) {
2514                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2515                         s->mb_intra= 0;
2516                         ff_mpeg4_set_direct_mv(s, 0, 0);
2517                     }
2518                     break;
2519                 case CANDIDATE_MB_TYPE_BIDIR:
2520                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2521                     s->mb_intra= 0;
2522                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2523                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2524                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2525                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2526                     break;
2527                 case CANDIDATE_MB_TYPE_BACKWARD:
2528                     s->mv_dir = MV_DIR_BACKWARD;
2529                     s->mb_intra= 0;
2530                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2531                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2532                     break;
2533                 case CANDIDATE_MB_TYPE_FORWARD:
2534                     s->mv_dir = MV_DIR_FORWARD;
2535                     s->mb_intra= 0;
2536                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2537                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2538 //                    printf(" %d %d ", motion_x, motion_y);
2539                     break;
2540                 case CANDIDATE_MB_TYPE_FORWARD_I:
2541                     s->mv_dir = MV_DIR_FORWARD;
2542                     s->mv_type = MV_TYPE_FIELD;
2543                     s->mb_intra= 0;
2544                     for(i=0; i<2; i++){
2545                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2546                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2547                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2548                     }
2549                     break;
2550                 case CANDIDATE_MB_TYPE_BACKWARD_I:
2551                     s->mv_dir = MV_DIR_BACKWARD;
2552                     s->mv_type = MV_TYPE_FIELD;
2553                     s->mb_intra= 0;
2554                     for(i=0; i<2; i++){
2555                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2556                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2557                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2558                     }
2559                     break;
2560                 case CANDIDATE_MB_TYPE_BIDIR_I:
2561                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2562                     s->mv_type = MV_TYPE_FIELD;
2563                     s->mb_intra= 0;
2564                     for(dir=0; dir<2; dir++){
2565                         for(i=0; i<2; i++){
2566                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2567                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2568                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2569                         }
2570                     }
2571                     break;
2572                 default:
2573                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
2574                 }
2575
2576                 encode_mb(s, motion_x, motion_y);
2577
2578                 // RAL: Update last macroblock type
2579                 s->last_mv_dir = s->mv_dir;
2580
2581                 if (ENABLE_ANY_H263_ENCODER &&
2582                     s->out_format == FMT_H263 && s->pict_type!=FF_B_TYPE)
2583                     ff_h263_update_motion_val(s);
2584
2585                 MPV_decode_mb(s, s->block);
2586             }
2587
2588             /* clean the MV table in IPS frames for direct mode in B frames */
2589             if(s->mb_intra /* && I,P,S_TYPE */){
2590                 s->p_mv_table[xy][0]=0;
2591                 s->p_mv_table[xy][1]=0;
2592             }
2593
2594             if(s->flags&CODEC_FLAG_PSNR){
2595                 int w= 16;
2596                 int h= 16;
2597
2598                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2599                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2600
2601                 s->current_picture.error[0] += sse(
2602                     s, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
2603                     s->dest[0], w, h, s->linesize);
2604                 s->current_picture.error[1] += sse(
2605                     s, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2606                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2607                 s->current_picture.error[2] += sse(
2608                     s, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2609                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2610             }
2611             if(s->loop_filter){
2612                 if(ENABLE_ANY_H263_ENCODER && s->out_format == FMT_H263)
2613                     ff_h263_loop_filter(s);
2614             }
2615 //printf("MB %d %d bits\n", s->mb_x+s->mb_y*s->mb_stride, put_bits_count(&s->pb));
2616         }
2617     }
2618
2619     //not beautiful here but we must write it before flushing so it has to be here
2620     if (ENABLE_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == FF_I_TYPE)
2621         msmpeg4_encode_ext_header(s);
2622
2623     write_slice_end(s);
2624
2625     /* Send the last GOB if RTP */
2626     if (s->avctx->rtp_callback) {
2627         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
2628         pdif = pbBufPtr(&s->pb) - s->ptr_lastgob;
2629         /* Call the RTP callback to send the last GOB */
2630         emms_c();
2631         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
2632     }
2633
2634     return 0;
2635 }
2636
2637 #define MERGE(field) dst->field += src->field; src->field=0
2638 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
2639     MERGE(me.scene_change_score);
2640     MERGE(me.mc_mb_var_sum_temp);
2641     MERGE(me.mb_var_sum_temp);
2642 }
2643
2644 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
2645     int i;
2646
2647     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
2648     MERGE(dct_count[1]);
2649     MERGE(mv_bits);
2650     MERGE(i_tex_bits);
2651     MERGE(p_tex_bits);
2652     MERGE(i_count);
2653     MERGE(f_count);
2654     MERGE(b_count);
2655     MERGE(skip_count);
2656     MERGE(misc_bits);
2657     MERGE(error_count);
2658     MERGE(padding_bug_score);
2659     MERGE(current_picture.error[0]);
2660     MERGE(current_picture.error[1]);
2661     MERGE(current_picture.error[2]);
2662
2663     if(dst->avctx->noise_reduction){
2664         for(i=0; i<64; i++){
2665             MERGE(dct_error_sum[0][i]);
2666             MERGE(dct_error_sum[1][i]);
2667         }
2668     }
2669
2670     assert(put_bits_count(&src->pb) % 8 ==0);
2671     assert(put_bits_count(&dst->pb) % 8 ==0);
2672     ff_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
2673     flush_put_bits(&dst->pb);
2674 }
2675
2676 static int estimate_qp(MpegEncContext *s, int dry_run){
2677     if (s->next_lambda){
2678         s->current_picture_ptr->quality=
2679         s->current_picture.quality = s->next_lambda;
2680         if(!dry_run) s->next_lambda= 0;
2681     } else if (!s->fixed_qscale) {
2682         s->current_picture_ptr->quality=
2683         s->current_picture.quality = ff_rate_estimate_qscale(s, dry_run);
2684         if (s->current_picture.quality < 0)
2685             return -1;
2686     }
2687
2688     if(s->adaptive_quant){
2689         switch(s->codec_id){
2690         case CODEC_ID_MPEG4:
2691             if (ENABLE_MPEG4_ENCODER)
2692                 ff_clean_mpeg4_qscales(s);
2693             break;
2694         case CODEC_ID_H263:
2695         case CODEC_ID_H263P:
2696         case CODEC_ID_FLV1:
2697             if (ENABLE_H263_ENCODER||ENABLE_H263P_ENCODER||ENABLE_FLV_ENCODER)
2698                 ff_clean_h263_qscales(s);
2699             break;
2700         }
2701
2702         s->lambda= s->lambda_table[0];
2703         //FIXME broken
2704     }else
2705         s->lambda= s->current_picture.quality;
2706 //printf("%d %d\n", s->avctx->global_quality, s->current_picture.quality);
2707     update_qscale(s);
2708     return 0;
2709 }
2710
2711 /* must be called before writing the header */
2712 static void set_frame_distances(MpegEncContext * s){
2713     assert(s->current_picture_ptr->pts != AV_NOPTS_VALUE);
2714     s->time= s->current_picture_ptr->pts*s->avctx->time_base.num;
2715
2716     if(s->pict_type==FF_B_TYPE){
2717         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
2718         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
2719     }else{
2720         s->pp_time= s->time - s->last_non_b_time;
2721         s->last_non_b_time= s->time;
2722         assert(s->picture_number==0 || s->pp_time > 0);
2723     }
2724 }
2725
2726 static int encode_picture(MpegEncContext *s, int picture_number)
2727 {
2728     int i;
2729     int bits;
2730
2731     s->picture_number = picture_number;
2732
2733     /* Reset the average MB variance */
2734     s->me.mb_var_sum_temp    =
2735     s->me.mc_mb_var_sum_temp = 0;
2736
2737     /* we need to initialize some time vars before we can encode b-frames */
2738     // RAL: Condition added for MPEG1VIDEO
2739     if (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->h263_msmpeg4))
2740         set_frame_distances(s);
2741     if(ENABLE_MPEG4_ENCODER && s->codec_id == CODEC_ID_MPEG4)
2742         ff_set_mpeg4_time(s);
2743
2744     s->me.scene_change_score=0;
2745
2746 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
2747
2748     if(s->pict_type==FF_I_TYPE){
2749         if(s->msmpeg4_version >= 3) s->no_rounding=1;
2750         else                        s->no_rounding=0;
2751     }else if(s->pict_type!=FF_B_TYPE){
2752         if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
2753             s->no_rounding ^= 1;
2754     }
2755
2756     if(s->flags & CODEC_FLAG_PASS2){
2757         if (estimate_qp(s,1) < 0)
2758             return -1;
2759         ff_get_2pass_fcode(s);
2760     }else if(!(s->flags & CODEC_FLAG_QSCALE)){
2761         if(s->pict_type==FF_B_TYPE)
2762             s->lambda= s->last_lambda_for[s->pict_type];
2763         else
2764             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
2765         update_qscale(s);
2766     }
2767
2768     s->mb_intra=0; //for the rate distortion & bit compare functions
2769     for(i=1; i<s->avctx->thread_count; i++){
2770         ff_update_duplicate_context(s->thread_context[i], s);
2771     }
2772
2773     if(ff_init_me(s)<0)
2774         return -1;
2775
2776     /* Estimate motion for every MB */
2777     if(s->pict_type != FF_I_TYPE){
2778         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
2779         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
2780         if(s->pict_type != FF_B_TYPE && s->avctx->me_threshold==0){
2781             if((s->avctx->pre_me && s->last_non_b_pict_type==FF_I_TYPE) || s->avctx->pre_me==2){
2782                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, (void**)&(s->thread_context[0]), NULL, s->avctx->thread_count, sizeof(void*));
2783             }
2784         }
2785
2786         s->avctx->execute(s->avctx, estimate_motion_thread, (void**)&(s->thread_context[0]), NULL, s->avctx->thread_count, sizeof(void*));
2787     }else /* if(s->pict_type == FF_I_TYPE) */{
2788         /* I-Frame */
2789         for(i=0; i<s->mb_stride*s->mb_height; i++)
2790             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
2791
2792         if(!s->fixed_qscale){
2793             /* finding spatial complexity for I-frame rate control */
2794             s->avctx->execute(s->avctx, mb_var_thread, (void**)&(s->thread_context[0]), NULL, s->avctx->thread_count, sizeof(void*));
2795         }
2796     }
2797     for(i=1; i<s->avctx->thread_count; i++){
2798         merge_context_after_me(s, s->thread_context[i]);
2799     }
2800     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
2801     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
2802     emms_c();
2803
2804     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == FF_P_TYPE){
2805         s->pict_type= FF_I_TYPE;
2806         for(i=0; i<s->mb_stride*s->mb_height; i++)
2807             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
2808 //printf("Scene change detected, encoding as I Frame %d %d\n", s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
2809     }
2810
2811     if(!s->umvplus){
2812         if(s->pict_type==FF_P_TYPE || s->pict_type==FF_S_TYPE) {
2813             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
2814
2815             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2816                 int a,b;
2817                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
2818                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
2819                 s->f_code= FFMAX3(s->f_code, a, b);
2820             }
2821
2822             ff_fix_long_p_mvs(s);
2823             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
2824             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2825                 int j;
2826                 for(i=0; i<2; i++){
2827                     for(j=0; j<2; j++)
2828                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
2829                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
2830                 }
2831             }
2832         }
2833
2834         if(s->pict_type==FF_B_TYPE){
2835             int a, b;
2836
2837             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
2838             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
2839             s->f_code = FFMAX(a, b);
2840
2841             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
2842             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
2843             s->b_code = FFMAX(a, b);
2844
2845             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
2846             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
2847             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
2848             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
2849             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2850                 int dir, j;
2851                 for(dir=0; dir<2; dir++){
2852                     for(i=0; i<2; i++){
2853                         for(j=0; j<2; j++){
2854                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
2855                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
2856                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
2857                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
2858                         }
2859                     }
2860                 }
2861             }
2862         }
2863     }
2864
2865     if (estimate_qp(s, 0) < 0)
2866         return -1;
2867
2868     if(s->qscale < 3 && s->max_qcoeff<=128 && s->pict_type==FF_I_TYPE && !(s->flags & CODEC_FLAG_QSCALE))
2869         s->qscale= 3; //reduce clipping problems
2870
2871     if (s->out_format == FMT_MJPEG) {
2872         /* for mjpeg, we do include qscale in the matrix */
2873         s->intra_matrix[0] = ff_mpeg1_default_intra_matrix[0];
2874         for(i=1;i<64;i++){
2875             int j= s->dsp.idct_permutation[i];
2876
2877             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
2878         }
2879         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
2880                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
2881         s->qscale= 8;
2882     }
2883
2884     //FIXME var duplication
2885     s->current_picture_ptr->key_frame=
2886     s->current_picture.key_frame= s->pict_type == FF_I_TYPE; //FIXME pic_ptr
2887     s->current_picture_ptr->pict_type=
2888     s->current_picture.pict_type= s->pict_type;
2889
2890     if(s->current_picture.key_frame)
2891         s->picture_in_gop_number=0;
2892
2893     s->last_bits= put_bits_count(&s->pb);
2894     switch(s->out_format) {
2895     case FMT_MJPEG:
2896         if (ENABLE_MJPEG_ENCODER)
2897             ff_mjpeg_encode_picture_header(s);
2898         break;
2899     case FMT_H261:
2900         if (ENABLE_H261_ENCODER)
2901             ff_h261_encode_picture_header(s, picture_number);
2902         break;
2903     case FMT_H263:
2904         if (ENABLE_WMV2_ENCODER && s->codec_id == CODEC_ID_WMV2)
2905             ff_wmv2_encode_picture_header(s, picture_number);
2906         else if (ENABLE_MSMPEG4_ENCODER && s->h263_msmpeg4)
2907             msmpeg4_encode_picture_header(s, picture_number);
2908         else if (ENABLE_MPEG4_ENCODER && s->h263_pred)
2909             mpeg4_encode_picture_header(s, picture_number);
2910         else if (ENABLE_RV10_ENCODER && s->codec_id == CODEC_ID_RV10)
2911             rv10_encode_picture_header(s, picture_number);
2912         else if (ENABLE_RV20_ENCODER && s->codec_id == CODEC_ID_RV20)
2913             rv20_encode_picture_header(s, picture_number);
2914         else if (ENABLE_FLV_ENCODER && s->codec_id == CODEC_ID_FLV1)
2915             ff_flv_encode_picture_header(s, picture_number);
2916         else if (ENABLE_ANY_H263_ENCODER)
2917             h263_encode_picture_header(s, picture_number);
2918         break;
2919     case FMT_MPEG1:
2920         if (ENABLE_MPEG1VIDEO_ENCODER || ENABLE_MPEG2VIDEO_ENCODER)
2921             mpeg1_encode_picture_header(s, picture_number);
2922         break;
2923     case FMT_H264:
2924         break;
2925     default:
2926         assert(0);
2927     }
2928     bits= put_bits_count(&s->pb);
2929     s->header_bits= bits - s->last_bits;
2930
2931     for(i=1; i<s->avctx->thread_count; i++){
2932         update_duplicate_context_after_me(s->thread_context[i], s);
2933     }
2934     s->avctx->execute(s->avctx, encode_thread, (void**)&(s->thread_context[0]), NULL, s->avctx->thread_count, sizeof(void*));
2935     for(i=1; i<s->avctx->thread_count; i++){
2936         merge_context_after_encode(s, s->thread_context[i]);
2937     }
2938     emms_c();
2939     return 0;
2940 }
2941
2942 void  denoise_dct_c(MpegEncContext *s, DCTELEM *block){
2943     const int intra= s->mb_intra;
2944     int i;
2945
2946     s->dct_count[intra]++;
2947
2948     for(i=0; i<64; i++){
2949         int level= block[i];
2950
2951         if(level){
2952             if(level>0){
2953                 s->dct_error_sum[intra][i] += level;
2954                 level -= s->dct_offset[intra][i];
2955                 if(level<0) level=0;
2956             }else{
2957                 s->dct_error_sum[intra][i] -= level;
2958                 level += s->dct_offset[intra][i];
2959                 if(level>0) level=0;
2960             }
2961             block[i]= level;
2962         }
2963     }
2964 }
2965
2966 int dct_quantize_trellis_c(MpegEncContext *s,
2967                         DCTELEM *block, int n,
2968                         int qscale, int *overflow){
2969     const int *qmat;
2970     const uint8_t *scantable= s->intra_scantable.scantable;
2971     const uint8_t *perm_scantable= s->intra_scantable.permutated;
2972     int max=0;
2973     unsigned int threshold1, threshold2;
2974     int bias=0;
2975     int run_tab[65];
2976     int level_tab[65];
2977     int score_tab[65];
2978     int survivor[65];
2979     int survivor_count;
2980     int last_run=0;
2981     int last_level=0;
2982     int last_score= 0;
2983     int last_i;
2984     int coeff[2][64];
2985     int coeff_count[64];
2986     int qmul, qadd, start_i, last_non_zero, i, dc;
2987     const int esc_length= s->ac_esc_length;
2988     uint8_t * length;
2989     uint8_t * last_length;
2990     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
2991
2992     s->dsp.fdct (block);
2993
2994     if(s->dct_error_sum)
2995         s->denoise_dct(s, block);
2996     qmul= qscale*16;
2997     qadd= ((qscale-1)|1)*8;
2998
2999     if (s->mb_intra) {
3000         int q;
3001         if (!s->h263_aic) {
3002             if (n < 4)
3003                 q = s->y_dc_scale;
3004             else
3005                 q = s->c_dc_scale;
3006             q = q << 3;
3007         } else{
3008             /* For AIC we skip quant/dequant of INTRADC */
3009             q = 1 << 3;
3010             qadd=0;
3011         }
3012
3013         /* note: block[0] is assumed to be positive */
3014         block[0] = (block[0] + (q >> 1)) / q;
3015         start_i = 1;
3016         last_non_zero = 0;
3017         qmat = s->q_intra_matrix[qscale];
3018         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3019             bias= 1<<(QMAT_SHIFT-1);
3020         length     = s->intra_ac_vlc_length;
3021         last_length= s->intra_ac_vlc_last_length;
3022     } else {
3023         start_i = 0;
3024         last_non_zero = -1;
3025         qmat = s->q_inter_matrix[qscale];
3026         length     = s->inter_ac_vlc_length;
3027         last_length= s->inter_ac_vlc_last_length;
3028     }
3029     last_i= start_i;
3030
3031     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3032     threshold2= (threshold1<<1);
3033
3034     for(i=63; i>=start_i; i--) {
3035         const int j = scantable[i];
3036         int level = block[j] * qmat[j];
3037
3038         if(((unsigned)(level+threshold1))>threshold2){
3039             last_non_zero = i;
3040             break;
3041         }
3042     }
3043
3044     for(i=start_i; i<=last_non_zero; i++) {
3045         const int j = scantable[i];
3046         int level = block[j] * qmat[j];
3047
3048 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3049 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3050         if(((unsigned)(level+threshold1))>threshold2){
3051             if(level>0){
3052                 level= (bias + level)>>QMAT_SHIFT;
3053                 coeff[0][i]= level;
3054                 coeff[1][i]= level-1;
3055 //                coeff[2][k]= level-2;
3056             }else{
3057                 level= (bias - level)>>QMAT_SHIFT;
3058                 coeff[0][i]= -level;
3059                 coeff[1][i]= -level+1;
3060 //                coeff[2][k]= -level+2;
3061             }
3062             coeff_count[i]= FFMIN(level, 2);
3063             assert(coeff_count[i]);
3064             max |=level;
3065         }else{
3066             coeff[0][i]= (level>>31)|1;
3067             coeff_count[i]= 1;
3068         }
3069     }
3070
3071     *overflow= s->max_qcoeff < max; //overflow might have happened
3072
3073     if(last_non_zero < start_i){
3074         memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3075         return last_non_zero;
3076     }
3077
3078     score_tab[start_i]= 0;
3079     survivor[0]= start_i;
3080     survivor_count= 1;
3081
3082     for(i=start_i; i<=last_non_zero; i++){
3083         int level_index, j, zero_distortion;
3084         int dct_coeff= FFABS(block[ scantable[i] ]);
3085         int best_score=256*256*256*120;
3086
3087         if (   s->dsp.fdct == fdct_ifast
3088 #ifndef FAAN_POSTSCALE
3089             || s->dsp.fdct == ff_faandct
3090 #endif
3091            )
3092             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3093         zero_distortion= dct_coeff*dct_coeff;
3094
3095         for(level_index=0; level_index < coeff_count[i]; level_index++){
3096             int distortion;
3097             int level= coeff[level_index][i];
3098             const int alevel= FFABS(level);
3099             int unquant_coeff;
3100
3101             assert(level);
3102
3103             if(s->out_format == FMT_H263){
3104                 unquant_coeff= alevel*qmul + qadd;
3105             }else{ //MPEG1
3106                 j= s->dsp.idct_permutation[ scantable[i] ]; //FIXME optimize
3107                 if(s->mb_intra){
3108                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3109                         unquant_coeff =   (unquant_coeff - 1) | 1;
3110                 }else{
3111                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3112                         unquant_coeff =   (unquant_coeff - 1) | 1;
3113                 }
3114                 unquant_coeff<<= 3;
3115             }
3116
3117             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3118             level+=64;
3119             if((level&(~127)) == 0){
3120                 for(j=survivor_count-1; j>=0; j--){
3121                     int run= i - survivor[j];
3122                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3123                     score += score_tab[i-run];
3124
3125                     if(score < best_score){
3126                         best_score= score;
3127                         run_tab[i+1]= run;
3128                         level_tab[i+1]= level-64;
3129                     }
3130                 }
3131
3132                 if(s->out_format == FMT_H263){
3133                     for(j=survivor_count-1; j>=0; j--){
3134                         int run= i - survivor[j];
3135                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3136                         score += score_tab[i-run];
3137                         if(score < last_score){
3138                             last_score= score;
3139                             last_run= run;
3140                             last_level= level-64;
3141                             last_i= i+1;
3142                         }
3143                     }
3144                 }
3145             }else{
3146                 distortion += esc_length*lambda;
3147                 for(j=survivor_count-1; j>=0; j--){
3148                     int run= i - survivor[j];
3149                     int score= distortion + score_tab[i-run];
3150
3151                     if(score < best_score){
3152                         best_score= score;
3153                         run_tab[i+1]= run;
3154                         level_tab[i+1]= level-64;
3155                     }
3156                 }
3157
3158                 if(s->out_format == FMT_H263){
3159                   for(j=survivor_count-1; j>=0; j--){
3160                         int run= i - survivor[j];
3161                         int score= distortion + score_tab[i-run];
3162                         if(score < last_score){
3163                             last_score= score;
3164                             last_run= run;
3165                             last_level= level-64;
3166                             last_i= i+1;
3167                         }
3168                     }
3169                 }
3170             }
3171         }
3172
3173         score_tab[i+1]= best_score;
3174
3175         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
3176         if(last_non_zero <= 27){
3177             for(; survivor_count; survivor_count--){
3178                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3179                     break;
3180             }
3181         }else{
3182             for(; survivor_count; survivor_count--){
3183                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3184                     break;
3185             }
3186         }
3187
3188         survivor[ survivor_count++ ]= i+1;
3189     }
3190
3191     if(s->out_format != FMT_H263){
3192         last_score= 256*256*256*120;
3193         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3194             int score= score_tab[i];
3195             if(i) score += lambda*2; //FIXME exacter?
3196
3197             if(score < last_score){
3198                 last_score= score;
3199                 last_i= i;
3200                 last_level= level_tab[i];
3201                 last_run= run_tab[i];
3202             }
3203         }
3204     }
3205
3206     s->coded_score[n] = last_score;
3207
3208     dc= FFABS(block[0]);
3209     last_non_zero= last_i - 1;
3210     memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3211
3212     if(last_non_zero < start_i)
3213         return last_non_zero;
3214
3215     if(last_non_zero == 0 && start_i == 0){
3216         int best_level= 0;
3217         int best_score= dc * dc;
3218
3219         for(i=0; i<coeff_count[0]; i++){
3220             int level= coeff[i][0];
3221             int alevel= FFABS(level);
3222             int unquant_coeff, score, distortion;
3223
3224             if(s->out_format == FMT_H263){
3225                     unquant_coeff= (alevel*qmul + qadd)>>3;
3226             }else{ //MPEG1
3227                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3228                     unquant_coeff =   (unquant_coeff - 1) | 1;
3229             }
3230             unquant_coeff = (unquant_coeff + 4) >> 3;
3231             unquant_coeff<<= 3 + 3;
3232
3233             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3234             level+=64;
3235             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3236             else                    score= distortion + esc_length*lambda;
3237
3238             if(score < best_score){
3239                 best_score= score;
3240                 best_level= level - 64;
3241             }
3242         }
3243         block[0]= best_level;
3244         s->coded_score[n] = best_score - dc*dc;
3245         if(best_level == 0) return -1;
3246         else                return last_non_zero;
3247     }
3248
3249     i= last_i;
3250     assert(last_level);
3251
3252     block[ perm_scantable[last_non_zero] ]= last_level;
3253     i -= last_run + 1;
3254
3255     for(; i>start_i; i -= run_tab[i] + 1){
3256         block[ perm_scantable[i-1] ]= level_tab[i];
3257     }
3258
3259     return last_non_zero;
3260 }
3261
3262 //#define REFINE_STATS 1
3263 static int16_t basis[64][64];
3264
3265 static void build_basis(uint8_t *perm){
3266     int i, j, x, y;
3267     emms_c();
3268     for(i=0; i<8; i++){
3269         for(j=0; j<8; j++){
3270             for(y=0; y<8; y++){
3271                 for(x=0; x<8; x++){
3272                     double s= 0.25*(1<<BASIS_SHIFT);
3273                     int index= 8*i + j;
3274                     int perm_index= perm[index];
3275                     if(i==0) s*= sqrt(0.5);
3276                     if(j==0) s*= sqrt(0.5);
3277                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3278                 }
3279             }
3280         }
3281     }
3282 }
3283
3284 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3285                         DCTELEM *block, int16_t *weight, DCTELEM *orig,
3286                         int n, int qscale){
3287     int16_t rem[64];
3288     DECLARE_ALIGNED_16(DCTELEM, d1[64]);
3289     const int *qmat;
3290     const uint8_t *scantable= s->intra_scantable.scantable;
3291     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3292 //    unsigned int threshold1, threshold2;
3293 //    int bias=0;
3294     int run_tab[65];
3295     int prev_run=0;
3296     int prev_level=0;
3297     int qmul, qadd, start_i, last_non_zero, i, dc;
3298     uint8_t * length;
3299     uint8_t * last_length;
3300     int lambda;
3301     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3302 #ifdef REFINE_STATS
3303 static int count=0;
3304 static int after_last=0;
3305 static int to_zero=0;
3306 static int from_zero=0;
3307 static int raise=0;
3308 static int lower=0;
3309 static int messed_sign=0;
3310 #endif
3311
3312     if(basis[0][0] == 0)
3313         build_basis(s->dsp.idct_permutation);
3314
3315     qmul= qscale*2;
3316     qadd= (qscale-1)|1;
3317     if (s->mb_intra) {
3318         if (!s->h263_aic) {
3319             if (n < 4)
3320                 q = s->y_dc_scale;
3321             else
3322                 q = s->c_dc_scale;
3323         } else{
3324             /* For AIC we skip quant/dequant of INTRADC */
3325             q = 1;
3326             qadd=0;
3327         }
3328         q <<= RECON_SHIFT-3;
3329         /* note: block[0] is assumed to be positive */
3330         dc= block[0]*q;
3331 //        block[0] = (block[0] + (q >> 1)) / q;
3332         start_i = 1;
3333         qmat = s->q_intra_matrix[qscale];
3334 //        if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3335 //            bias= 1<<(QMAT_SHIFT-1);
3336         length     = s->intra_ac_vlc_length;
3337         last_length= s->intra_ac_vlc_last_length;
3338     } else {
3339         dc= 0;
3340         start_i = 0;
3341         qmat = s->q_inter_matrix[qscale];
3342         length     = s->inter_ac_vlc_length;
3343         last_length= s->inter_ac_vlc_last_length;
3344     }
3345     last_non_zero = s->block_last_index[n];
3346
3347 #ifdef REFINE_STATS
3348 {START_TIMER
3349 #endif
3350     dc += (1<<(RECON_SHIFT-1));
3351     for(i=0; i<64; i++){
3352         rem[i]= dc - (orig[i]<<RECON_SHIFT); //FIXME  use orig dirrectly instead of copying to rem[]
3353     }
3354 #ifdef REFINE_STATS
3355 STOP_TIMER("memset rem[]")}
3356 #endif
3357     sum=0;
3358     for(i=0; i<64; i++){
3359         int one= 36;
3360         int qns=4;
3361         int w;
3362
3363         w= FFABS(weight[i]) + qns*one;
3364         w= 15 + (48*qns*one + w/2)/w; // 16 .. 63
3365
3366         weight[i] = w;
3367 //        w=weight[i] = (63*qns + (w/2)) / w;
3368
3369         assert(w>0);
3370         assert(w<(1<<6));
3371         sum += w*w;
3372     }
3373     lambda= sum*(uint64_t)s->lambda2 >> (FF_LAMBDA_SHIFT - 6 + 6 + 6 + 6);
3374 #ifdef REFINE_STATS
3375 {START_TIMER
3376 #endif
3377     run=0;
3378     rle_index=0;