h264: fix loopfilter with threading at slice boundaries.
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of Libav.
9  *
10  * Libav is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * Libav is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with Libav; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include "libavutil/intmath.h"
31 #include "avcodec.h"
32 #include "dsputil.h"
33 #include "mpegvideo.h"
34 #include "mpegvideo_common.h"
35 #include "h263.h"
36 #include "mjpegenc.h"
37 #include "msmpeg4.h"
38 #include "faandct.h"
39 #include "aandcttab.h"
40 #include "flv.h"
41 #include "mpeg4video.h"
42 #include "internal.h"
43 #include <limits.h>
44
45 //#undef NDEBUG
46 //#include <assert.h>
47
48 static int encode_picture(MpegEncContext *s, int picture_number);
49 static int dct_quantize_refine(MpegEncContext *s, DCTELEM *block, int16_t *weight, DCTELEM *orig, int n, int qscale);
50 static int sse_mb(MpegEncContext *s);
51 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block);
52 static int dct_quantize_trellis_c(MpegEncContext *s, DCTELEM *block, int n, int qscale, int *overflow);
53
54 /* enable all paranoid tests for rounding, overflows, etc... */
55 //#define PARANOID
56
57 //#define DEBUG
58
59 static uint8_t default_mv_penalty[MAX_FCODE+1][MAX_MV*2+1];
60 static uint8_t default_fcode_tab[MAX_MV*2+1];
61
62 void ff_convert_matrix(DSPContext *dsp, int (*qmat)[64], uint16_t (*qmat16)[2][64],
63                            const uint16_t *quant_matrix, int bias, int qmin, int qmax, int intra)
64 {
65     int qscale;
66     int shift=0;
67
68     for(qscale=qmin; qscale<=qmax; qscale++){
69         int i;
70         if (dsp->fdct == ff_jpeg_fdct_islow
71 #ifdef FAAN_POSTSCALE
72             || dsp->fdct == ff_faandct
73 #endif
74             ) {
75             for(i=0;i<64;i++) {
76                 const int j= dsp->idct_permutation[i];
77                 /* 16 <= qscale * quant_matrix[i] <= 7905 */
78                 /* 19952             <= ff_aanscales[i] * qscale * quant_matrix[i]               <= 249205026 */
79                 /* (1 << 36) / 19952 >= (1 << 36) / (ff_aanscales[i] * qscale * quant_matrix[i]) >= (1 << 36) / 249205026 */
80                 /* 3444240           >= (1 << 36) / (ff_aanscales[i] * qscale * quant_matrix[i]) >= 275 */
81
82                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
83                                 (qscale * quant_matrix[j]));
84             }
85         } else if (dsp->fdct == fdct_ifast
86 #ifndef FAAN_POSTSCALE
87                    || dsp->fdct == ff_faandct
88 #endif
89                    ) {
90             for(i=0;i<64;i++) {
91                 const int j= dsp->idct_permutation[i];
92                 /* 16 <= qscale * quant_matrix[i] <= 7905 */
93                 /* 19952             <= ff_aanscales[i] * qscale * quant_matrix[i]               <= 249205026 */
94                 /* (1 << 36) / 19952 >= (1 << 36) / (ff_aanscales[i] * qscale * quant_matrix[i]) >= (1<<36)/249205026 */
95                 /* 3444240           >= (1 << 36) / (ff_aanscales[i] * qscale * quant_matrix[i]) >= 275 */
96
97                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) /
98                                 (ff_aanscales[i] * qscale * quant_matrix[j]));
99             }
100         } else {
101             for(i=0;i<64;i++) {
102                 const int j= dsp->idct_permutation[i];
103                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
104                    So 16           <= qscale * quant_matrix[i]             <= 7905
105                    so (1<<19) / 16 >= (1<<19) / (qscale * quant_matrix[i]) >= (1<<19) / 7905
106                    so 32768        >= (1<<19) / (qscale * quant_matrix[i]) >= 67
107                 */
108                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / (qscale * quant_matrix[j]));
109 //                qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) / (qscale * quant_matrix[i]);
110                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / (qscale * quant_matrix[j]);
111
112                 if(qmat16[qscale][0][i]==0 || qmat16[qscale][0][i]==128*256) qmat16[qscale][0][i]=128*256-1;
113                 qmat16[qscale][1][i]= ROUNDED_DIV(bias<<(16-QUANT_BIAS_SHIFT), qmat16[qscale][0][i]);
114             }
115         }
116
117         for(i=intra; i<64; i++){
118             int64_t max= 8191;
119             if (dsp->fdct == fdct_ifast
120 #ifndef FAAN_POSTSCALE
121                    || dsp->fdct == ff_faandct
122 #endif
123                    ) {
124                 max = (8191LL*ff_aanscales[i]) >> 14;
125             }
126             while(((max * qmat[qscale][i]) >> shift) > INT_MAX){
127                 shift++;
128             }
129         }
130     }
131     if(shift){
132         av_log(NULL, AV_LOG_INFO, "Warning, QMAT_SHIFT is larger than %d, overflows possible\n", QMAT_SHIFT - shift);
133     }
134 }
135
136 static inline void update_qscale(MpegEncContext *s){
137     s->qscale= (s->lambda*139 + FF_LAMBDA_SCALE*64) >> (FF_LAMBDA_SHIFT + 7);
138     s->qscale= av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
139
140     s->lambda2= (s->lambda*s->lambda + FF_LAMBDA_SCALE/2) >> FF_LAMBDA_SHIFT;
141 }
142
143 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix){
144     int i;
145
146     if(matrix){
147         put_bits(pb, 1, 1);
148         for(i=0;i<64;i++) {
149             put_bits(pb, 8, matrix[ ff_zigzag_direct[i] ]);
150         }
151     }else
152         put_bits(pb, 1, 0);
153 }
154
155 /**
156  * init s->current_picture.qscale_table from s->lambda_table
157  */
158 void ff_init_qscale_tab(MpegEncContext *s){
159     int8_t * const qscale_table= s->current_picture.qscale_table;
160     int i;
161
162     for(i=0; i<s->mb_num; i++){
163         unsigned int lam= s->lambda_table[ s->mb_index2xy[i] ];
164         int qp= (lam*139 + FF_LAMBDA_SCALE*64) >> (FF_LAMBDA_SHIFT + 7);
165         qscale_table[ s->mb_index2xy[i] ]= av_clip(qp, s->avctx->qmin, s->avctx->qmax);
166     }
167 }
168
169 static void copy_picture_attributes(MpegEncContext *s, AVFrame *dst, AVFrame *src){
170     int i;
171
172     dst->pict_type              = src->pict_type;
173     dst->quality                = src->quality;
174     dst->coded_picture_number   = src->coded_picture_number;
175     dst->display_picture_number = src->display_picture_number;
176 //    dst->reference              = src->reference;
177     dst->pts                    = src->pts;
178     dst->interlaced_frame       = src->interlaced_frame;
179     dst->top_field_first        = src->top_field_first;
180
181     if(s->avctx->me_threshold){
182         if(!src->motion_val[0])
183             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_val not set!\n");
184         if(!src->mb_type)
185             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.mb_type not set!\n");
186         if(!src->ref_index[0])
187             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.ref_index not set!\n");
188         if(src->motion_subsample_log2 != dst->motion_subsample_log2)
189             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_subsample_log2 doesn't match! (%d!=%d)\n",
190             src->motion_subsample_log2, dst->motion_subsample_log2);
191
192         memcpy(dst->mb_type, src->mb_type, s->mb_stride * s->mb_height * sizeof(dst->mb_type[0]));
193
194         for(i=0; i<2; i++){
195             int stride= ((16*s->mb_width )>>src->motion_subsample_log2) + 1;
196             int height= ((16*s->mb_height)>>src->motion_subsample_log2);
197
198             if(src->motion_val[i] && src->motion_val[i] != dst->motion_val[i]){
199                 memcpy(dst->motion_val[i], src->motion_val[i], 2*stride*height*sizeof(int16_t));
200             }
201             if(src->ref_index[i] && src->ref_index[i] != dst->ref_index[i]){
202                 memcpy(dst->ref_index[i], src->ref_index[i], s->mb_stride*4*s->mb_height*sizeof(int8_t));
203             }
204         }
205     }
206 }
207
208 static void update_duplicate_context_after_me(MpegEncContext *dst, MpegEncContext *src){
209 #define COPY(a) dst->a= src->a
210     COPY(pict_type);
211     COPY(current_picture);
212     COPY(f_code);
213     COPY(b_code);
214     COPY(qscale);
215     COPY(lambda);
216     COPY(lambda2);
217     COPY(picture_in_gop_number);
218     COPY(gop_picture_number);
219     COPY(frame_pred_frame_dct); //FIXME don't set in encode_header
220     COPY(progressive_frame); //FIXME don't set in encode_header
221     COPY(partitioned_frame); //FIXME don't set in encode_header
222 #undef COPY
223 }
224
225 /**
226  * sets the given MpegEncContext to defaults for encoding.
227  * the changed fields will not depend upon the prior state of the MpegEncContext.
228  */
229 static void MPV_encode_defaults(MpegEncContext *s){
230     int i;
231     MPV_common_defaults(s);
232
233     for(i=-16; i<16; i++){
234         default_fcode_tab[i + MAX_MV]= 1;
235     }
236     s->me.mv_penalty= default_mv_penalty;
237     s->fcode_tab= default_fcode_tab;
238 }
239
240 /* init video encoder */
241 av_cold int MPV_encode_init(AVCodecContext *avctx)
242 {
243     MpegEncContext *s = avctx->priv_data;
244     int i;
245     int chroma_h_shift, chroma_v_shift;
246
247     MPV_encode_defaults(s);
248
249     switch (avctx->codec_id) {
250     case CODEC_ID_MPEG2VIDEO:
251         if(avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P){
252             av_log(avctx, AV_LOG_ERROR, "only YUV420 and YUV422 are supported\n");
253             return -1;
254         }
255         break;
256     case CODEC_ID_LJPEG:
257         if(avctx->pix_fmt != PIX_FMT_YUVJ420P && avctx->pix_fmt != PIX_FMT_YUVJ422P && avctx->pix_fmt != PIX_FMT_YUVJ444P && avctx->pix_fmt != PIX_FMT_BGRA &&
258            ((avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P && avctx->pix_fmt != PIX_FMT_YUV444P) || avctx->strict_std_compliance>FF_COMPLIANCE_UNOFFICIAL)){
259             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in LJPEG\n");
260             return -1;
261         }
262         break;
263     case CODEC_ID_MJPEG:
264         if(avctx->pix_fmt != PIX_FMT_YUVJ420P && avctx->pix_fmt != PIX_FMT_YUVJ422P &&
265            ((avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P) || avctx->strict_std_compliance>FF_COMPLIANCE_UNOFFICIAL)){
266             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
267             return -1;
268         }
269         break;
270     default:
271         if(avctx->pix_fmt != PIX_FMT_YUV420P){
272             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
273             return -1;
274         }
275     }
276
277     switch (avctx->pix_fmt) {
278     case PIX_FMT_YUVJ422P:
279     case PIX_FMT_YUV422P:
280         s->chroma_format = CHROMA_422;
281         break;
282     case PIX_FMT_YUVJ420P:
283     case PIX_FMT_YUV420P:
284     default:
285         s->chroma_format = CHROMA_420;
286         break;
287     }
288
289     s->bit_rate = avctx->bit_rate;
290     s->width = avctx->width;
291     s->height = avctx->height;
292     if(avctx->gop_size > 600 && avctx->strict_std_compliance>FF_COMPLIANCE_EXPERIMENTAL){
293         av_log(avctx, AV_LOG_ERROR, "Warning keyframe interval too large! reducing it ...\n");
294         avctx->gop_size=600;
295     }
296     s->gop_size = avctx->gop_size;
297     s->avctx = avctx;
298     s->flags= avctx->flags;
299     s->flags2= avctx->flags2;
300     s->max_b_frames= avctx->max_b_frames;
301     s->codec_id= avctx->codec->id;
302     s->luma_elim_threshold  = avctx->luma_elim_threshold;
303     s->chroma_elim_threshold= avctx->chroma_elim_threshold;
304     s->strict_std_compliance= avctx->strict_std_compliance;
305     s->data_partitioning= avctx->flags & CODEC_FLAG_PART;
306     s->quarter_sample= (avctx->flags & CODEC_FLAG_QPEL)!=0;
307     s->mpeg_quant= avctx->mpeg_quant;
308     s->rtp_mode= !!avctx->rtp_payload_size;
309     s->intra_dc_precision= avctx->intra_dc_precision;
310     s->user_specified_pts = AV_NOPTS_VALUE;
311
312     if (s->gop_size <= 1) {
313         s->intra_only = 1;
314         s->gop_size = 12;
315     } else {
316         s->intra_only = 0;
317     }
318
319     s->me_method = avctx->me_method;
320
321     /* Fixed QSCALE */
322     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
323
324     s->adaptive_quant= (   s->avctx->lumi_masking
325                         || s->avctx->dark_masking
326                         || s->avctx->temporal_cplx_masking
327                         || s->avctx->spatial_cplx_masking
328                         || s->avctx->p_masking
329                         || s->avctx->border_masking
330                         || (s->flags&CODEC_FLAG_QP_RD))
331                        && !s->fixed_qscale;
332
333     s->obmc= !!(s->flags & CODEC_FLAG_OBMC);
334     s->loop_filter= !!(s->flags & CODEC_FLAG_LOOP_FILTER);
335     s->alternate_scan= !!(s->flags & CODEC_FLAG_ALT_SCAN);
336     s->intra_vlc_format= !!(s->flags2 & CODEC_FLAG2_INTRA_VLC);
337     s->q_scale_type= !!(s->flags2 & CODEC_FLAG2_NON_LINEAR_QUANT);
338
339     if(avctx->rc_max_rate && !avctx->rc_buffer_size){
340         av_log(avctx, AV_LOG_ERROR, "a vbv buffer size is needed, for encoding with a maximum bitrate\n");
341         return -1;
342     }
343
344     if(avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate){
345         av_log(avctx, AV_LOG_INFO, "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
346     }
347
348     if(avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate){
349         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
350         return -1;
351     }
352
353     if(avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate){
354         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
355         return -1;
356     }
357
358     if(avctx->rc_max_rate && avctx->rc_max_rate == avctx->bit_rate && avctx->rc_max_rate != avctx->rc_min_rate){
359         av_log(avctx, AV_LOG_INFO, "impossible bitrate constraints, this will fail\n");
360     }
361
362     if(avctx->rc_buffer_size && avctx->bit_rate*(int64_t)avctx->time_base.num > avctx->rc_buffer_size * (int64_t)avctx->time_base.den){
363         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
364         return -1;
365     }
366
367     if(!s->fixed_qscale && avctx->bit_rate*av_q2d(avctx->time_base) > avctx->bit_rate_tolerance){
368         av_log(avctx, AV_LOG_ERROR, "bitrate tolerance too small for bitrate\n");
369         return -1;
370     }
371
372     if(   s->avctx->rc_max_rate && s->avctx->rc_min_rate == s->avctx->rc_max_rate
373        && (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO)
374        && 90000LL * (avctx->rc_buffer_size-1) > s->avctx->rc_max_rate*0xFFFFLL){
375
376         av_log(avctx, AV_LOG_INFO, "Warning vbv_delay will be set to 0xFFFF (=VBR) as the specified vbv buffer is too large for the given bitrate!\n");
377     }
378
379     if((s->flags & CODEC_FLAG_4MV) && s->codec_id != CODEC_ID_MPEG4
380        && s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P && s->codec_id != CODEC_ID_FLV1){
381         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
382         return -1;
383     }
384
385     if(s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE){
386         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with simple mb decision\n");
387         return -1;
388     }
389
390     if(s->obmc && s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P){
391         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with H263(+)\n");
392         return -1;
393     }
394
395     if(s->quarter_sample && s->codec_id != CODEC_ID_MPEG4){
396         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
397         return -1;
398     }
399
400     if(s->data_partitioning && s->codec_id != CODEC_ID_MPEG4){
401         av_log(avctx, AV_LOG_ERROR, "data partitioning not supported by codec\n");
402         return -1;
403     }
404
405     if(s->max_b_frames && s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG1VIDEO && s->codec_id != CODEC_ID_MPEG2VIDEO){
406         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
407         return -1;
408     }
409
410     if ((s->codec_id == CODEC_ID_MPEG4 || s->codec_id == CODEC_ID_H263 ||
411          s->codec_id == CODEC_ID_H263P) &&
412         (avctx->sample_aspect_ratio.num > 255 || avctx->sample_aspect_ratio.den > 255)) {
413         av_log(avctx, AV_LOG_ERROR, "Invalid pixel aspect ratio %i/%i, limit is 255/255\n",
414                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
415         return -1;
416     }
417
418     if((s->flags & (CODEC_FLAG_INTERLACED_DCT|CODEC_FLAG_INTERLACED_ME|CODEC_FLAG_ALT_SCAN))
419        && s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG2VIDEO){
420         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
421         return -1;
422     }
423
424     if(s->mpeg_quant && s->codec_id != CODEC_ID_MPEG4){ //FIXME mpeg2 uses that too
425         av_log(avctx, AV_LOG_ERROR, "mpeg2 style quantization not supported by codec\n");
426         return -1;
427     }
428
429     if((s->flags & CODEC_FLAG_CBP_RD) && !avctx->trellis){
430         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
431         return -1;
432     }
433
434     if((s->flags & CODEC_FLAG_QP_RD) && s->avctx->mb_decision != FF_MB_DECISION_RD){
435         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
436         return -1;
437     }
438
439     if(s->avctx->scenechange_threshold < 1000000000 && (s->flags & CODEC_FLAG_CLOSED_GOP)){
440         av_log(avctx, AV_LOG_ERROR, "closed gop with scene change detection are not supported yet, set threshold to 1000000000\n");
441         return -1;
442     }
443
444     if((s->flags2 & CODEC_FLAG2_INTRA_VLC) && s->codec_id != CODEC_ID_MPEG2VIDEO){
445         av_log(avctx, AV_LOG_ERROR, "intra vlc table not supported by codec\n");
446         return -1;
447     }
448
449     if(s->flags & CODEC_FLAG_LOW_DELAY){
450         if (s->codec_id != CODEC_ID_MPEG2VIDEO){
451             av_log(avctx, AV_LOG_ERROR, "low delay forcing is only available for mpeg2\n");
452             return -1;
453         }
454         if (s->max_b_frames != 0){
455             av_log(avctx, AV_LOG_ERROR, "b frames cannot be used with low delay\n");
456             return -1;
457         }
458     }
459
460     if(s->q_scale_type == 1){
461         if(s->codec_id != CODEC_ID_MPEG2VIDEO){
462             av_log(avctx, AV_LOG_ERROR, "non linear quant is only available for mpeg2\n");
463             return -1;
464         }
465         if(avctx->qmax > 12){
466             av_log(avctx, AV_LOG_ERROR, "non linear quant only supports qmax <= 12 currently\n");
467             return -1;
468         }
469     }
470
471     if(s->avctx->thread_count > 1 && s->codec_id != CODEC_ID_MPEG4
472        && s->codec_id != CODEC_ID_MPEG1VIDEO && s->codec_id != CODEC_ID_MPEG2VIDEO
473        && (s->codec_id != CODEC_ID_H263P || !(s->flags & CODEC_FLAG_H263P_SLICE_STRUCT))){
474         av_log(avctx, AV_LOG_ERROR, "multi threaded encoding not supported by codec\n");
475         return -1;
476     }
477
478     if(s->avctx->thread_count < 1){
479         av_log(avctx, AV_LOG_ERROR, "automatic thread number detection not supported by codec, patch welcome\n");
480         return -1;
481     }
482
483     if(s->avctx->thread_count > 1)
484         s->rtp_mode= 1;
485
486     if(!avctx->time_base.den || !avctx->time_base.num){
487         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
488         return -1;
489     }
490
491     i= (INT_MAX/2+128)>>8;
492     if(avctx->me_threshold >= i){
493         av_log(avctx, AV_LOG_ERROR, "me_threshold too large, max is %d\n", i - 1);
494         return -1;
495     }
496     if(avctx->mb_threshold >= i){
497         av_log(avctx, AV_LOG_ERROR, "mb_threshold too large, max is %d\n", i - 1);
498         return -1;
499     }
500
501     if(avctx->b_frame_strategy && (avctx->flags&CODEC_FLAG_PASS2)){
502         av_log(avctx, AV_LOG_INFO, "notice: b_frame_strategy only affects the first pass\n");
503         avctx->b_frame_strategy = 0;
504     }
505
506     i= av_gcd(avctx->time_base.den, avctx->time_base.num);
507     if(i > 1){
508         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
509         avctx->time_base.den /= i;
510         avctx->time_base.num /= i;
511 //        return -1;
512     }
513
514     if(s->mpeg_quant || s->codec_id==CODEC_ID_MPEG1VIDEO || s->codec_id==CODEC_ID_MPEG2VIDEO || s->codec_id==CODEC_ID_MJPEG){
515         s->intra_quant_bias= 3<<(QUANT_BIAS_SHIFT-3); //(a + x*3/8)/x
516         s->inter_quant_bias= 0;
517     }else{
518         s->intra_quant_bias=0;
519         s->inter_quant_bias=-(1<<(QUANT_BIAS_SHIFT-2)); //(a - x/4)/x
520     }
521
522     if(avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
523         s->intra_quant_bias= avctx->intra_quant_bias;
524     if(avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
525         s->inter_quant_bias= avctx->inter_quant_bias;
526
527     avcodec_get_chroma_sub_sample(avctx->pix_fmt, &chroma_h_shift, &chroma_v_shift);
528
529     if(avctx->codec_id == CODEC_ID_MPEG4 && s->avctx->time_base.den > (1<<16)-1){
530         av_log(avctx, AV_LOG_ERROR, "timebase %d/%d not supported by MPEG 4 standard, "
531                "the maximum admitted value for the timebase denominator is %d\n",
532                s->avctx->time_base.num, s->avctx->time_base.den, (1<<16)-1);
533         return -1;
534     }
535     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
536
537     switch(avctx->codec->id) {
538     case CODEC_ID_MPEG1VIDEO:
539         s->out_format = FMT_MPEG1;
540         s->low_delay= !!(s->flags & CODEC_FLAG_LOW_DELAY);
541         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
542         break;
543     case CODEC_ID_MPEG2VIDEO:
544         s->out_format = FMT_MPEG1;
545         s->low_delay= !!(s->flags & CODEC_FLAG_LOW_DELAY);
546         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
547         s->rtp_mode= 1;
548         break;
549     case CODEC_ID_LJPEG:
550     case CODEC_ID_MJPEG:
551         s->out_format = FMT_MJPEG;
552         s->intra_only = 1; /* force intra only for jpeg */
553         if(avctx->codec->id == CODEC_ID_LJPEG && avctx->pix_fmt == PIX_FMT_BGRA){
554             s->mjpeg_vsample[0] = s->mjpeg_hsample[0] =
555             s->mjpeg_vsample[1] = s->mjpeg_hsample[1] =
556             s->mjpeg_vsample[2] = s->mjpeg_hsample[2] = 1;
557         }else{
558             s->mjpeg_vsample[0] = 2;
559             s->mjpeg_vsample[1] = 2>>chroma_v_shift;
560             s->mjpeg_vsample[2] = 2>>chroma_v_shift;
561             s->mjpeg_hsample[0] = 2;
562             s->mjpeg_hsample[1] = 2>>chroma_h_shift;
563             s->mjpeg_hsample[2] = 2>>chroma_h_shift;
564         }
565         if (!(CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER)
566             || ff_mjpeg_encode_init(s) < 0)
567             return -1;
568         avctx->delay=0;
569         s->low_delay=1;
570         break;
571     case CODEC_ID_H261:
572         if (!CONFIG_H261_ENCODER)  return -1;
573         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
574             av_log(avctx, AV_LOG_ERROR, "The specified picture size of %dx%d is not valid for the H.261 codec.\nValid sizes are 176x144, 352x288\n", s->width, s->height);
575             return -1;
576         }
577         s->out_format = FMT_H261;
578         avctx->delay=0;
579         s->low_delay=1;
580         break;
581     case CODEC_ID_H263:
582         if (!CONFIG_H263_ENCODER)  return -1;
583         if (ff_match_2uint16(h263_format, FF_ARRAY_ELEMS(h263_format), s->width, s->height) == 8) {
584             av_log(avctx, AV_LOG_INFO, "The specified picture size of %dx%d is not valid for the H.263 codec.\nValid sizes are 128x96, 176x144, 352x288, 704x576, and 1408x1152. Try H.263+.\n", s->width, s->height);
585             return -1;
586         }
587         s->out_format = FMT_H263;
588         s->obmc= (avctx->flags & CODEC_FLAG_OBMC) ? 1:0;
589         avctx->delay=0;
590         s->low_delay=1;
591         break;
592     case CODEC_ID_H263P:
593         s->out_format = FMT_H263;
594         s->h263_plus = 1;
595         /* Fx */
596         s->umvplus = (avctx->flags & CODEC_FLAG_H263P_UMV) ? 1:0;
597         s->h263_aic= (avctx->flags & CODEC_FLAG_AC_PRED) ? 1:0;
598         s->modified_quant= s->h263_aic;
599         s->alt_inter_vlc= (avctx->flags & CODEC_FLAG_H263P_AIV) ? 1:0;
600         s->obmc= (avctx->flags & CODEC_FLAG_OBMC) ? 1:0;
601         s->loop_filter= (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1:0;
602         s->unrestricted_mv= s->obmc || s->loop_filter || s->umvplus;
603         s->h263_slice_structured= (s->flags & CODEC_FLAG_H263P_SLICE_STRUCT) ? 1:0;
604
605         /* /Fx */
606         /* These are just to be sure */
607         avctx->delay=0;
608         s->low_delay=1;
609         break;
610     case CODEC_ID_FLV1:
611         s->out_format = FMT_H263;
612         s->h263_flv = 2; /* format = 1; 11-bit codes */
613         s->unrestricted_mv = 1;
614         s->rtp_mode=0; /* don't allow GOB */
615         avctx->delay=0;
616         s->low_delay=1;
617         break;
618     case CODEC_ID_RV10:
619         s->out_format = FMT_H263;
620         avctx->delay=0;
621         s->low_delay=1;
622         break;
623     case CODEC_ID_RV20:
624         s->out_format = FMT_H263;
625         avctx->delay=0;
626         s->low_delay=1;
627         s->modified_quant=1;
628         s->h263_aic=1;
629         s->h263_plus=1;
630         s->loop_filter=1;
631         s->unrestricted_mv= 0;
632         break;
633     case CODEC_ID_MPEG4:
634         s->out_format = FMT_H263;
635         s->h263_pred = 1;
636         s->unrestricted_mv = 1;
637         s->low_delay= s->max_b_frames ? 0 : 1;
638         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
639         break;
640     case CODEC_ID_MSMPEG4V1:
641         s->out_format = FMT_H263;
642         s->h263_msmpeg4 = 1;
643         s->h263_pred = 1;
644         s->unrestricted_mv = 1;
645         s->msmpeg4_version= 1;
646         avctx->delay=0;
647         s->low_delay=1;
648         break;
649     case CODEC_ID_MSMPEG4V2:
650         s->out_format = FMT_H263;
651         s->h263_msmpeg4 = 1;
652         s->h263_pred = 1;
653         s->unrestricted_mv = 1;
654         s->msmpeg4_version= 2;
655         avctx->delay=0;
656         s->low_delay=1;
657         break;
658     case CODEC_ID_MSMPEG4V3:
659         s->out_format = FMT_H263;
660         s->h263_msmpeg4 = 1;
661         s->h263_pred = 1;
662         s->unrestricted_mv = 1;
663         s->msmpeg4_version= 3;
664         s->flipflop_rounding=1;
665         avctx->delay=0;
666         s->low_delay=1;
667         break;
668     case CODEC_ID_WMV1:
669         s->out_format = FMT_H263;
670         s->h263_msmpeg4 = 1;
671         s->h263_pred = 1;
672         s->unrestricted_mv = 1;
673         s->msmpeg4_version= 4;
674         s->flipflop_rounding=1;
675         avctx->delay=0;
676         s->low_delay=1;
677         break;
678     case CODEC_ID_WMV2:
679         s->out_format = FMT_H263;
680         s->h263_msmpeg4 = 1;
681         s->h263_pred = 1;
682         s->unrestricted_mv = 1;
683         s->msmpeg4_version= 5;
684         s->flipflop_rounding=1;
685         avctx->delay=0;
686         s->low_delay=1;
687         break;
688     default:
689         return -1;
690     }
691
692     avctx->has_b_frames= !s->low_delay;
693
694     s->encoding = 1;
695
696     s->progressive_frame=
697     s->progressive_sequence= !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT|CODEC_FLAG_INTERLACED_ME|CODEC_FLAG_ALT_SCAN));
698
699     /* init */
700     if (MPV_common_init(s) < 0)
701         return -1;
702
703     if(!s->dct_quantize)
704         s->dct_quantize = dct_quantize_c;
705     if(!s->denoise_dct)
706         s->denoise_dct = denoise_dct_c;
707     s->fast_dct_quantize = s->dct_quantize;
708     if(avctx->trellis)
709         s->dct_quantize = dct_quantize_trellis_c;
710
711     if((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
712         s->chroma_qscale_table= ff_h263_chroma_qscale_table;
713
714     s->quant_precision=5;
715
716     ff_set_cmp(&s->dsp, s->dsp.ildct_cmp, s->avctx->ildct_cmp);
717     ff_set_cmp(&s->dsp, s->dsp.frame_skip_cmp, s->avctx->frame_skip_cmp);
718
719     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
720         ff_h261_encode_init(s);
721     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
722         h263_encode_init(s);
723     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
724         ff_msmpeg4_encode_init(s);
725     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
726         && s->out_format == FMT_MPEG1)
727         ff_mpeg1_encode_init(s);
728
729     /* init q matrix */
730     for(i=0;i<64;i++) {
731         int j= s->dsp.idct_permutation[i];
732         if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4 && s->mpeg_quant){
733             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
734             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
735         }else if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
736             s->intra_matrix[j] =
737             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
738         }else
739         { /* mpeg1/2 */
740             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
741             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
742         }
743         if(s->avctx->intra_matrix)
744             s->intra_matrix[j] = s->avctx->intra_matrix[i];
745         if(s->avctx->inter_matrix)
746             s->inter_matrix[j] = s->avctx->inter_matrix[i];
747     }
748
749     /* precompute matrix */
750     /* for mjpeg, we do include qscale in the matrix */
751     if (s->out_format != FMT_MJPEG) {
752         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
753                        s->intra_matrix, s->intra_quant_bias, avctx->qmin, 31, 1);
754         ff_convert_matrix(&s->dsp, s->q_inter_matrix, s->q_inter_matrix16,
755                        s->inter_matrix, s->inter_quant_bias, avctx->qmin, 31, 0);
756     }
757
758     if(ff_rate_control_init(s) < 0)
759         return -1;
760
761     return 0;
762 }
763
764 av_cold int MPV_encode_end(AVCodecContext *avctx)
765 {
766     MpegEncContext *s = avctx->priv_data;
767
768     ff_rate_control_uninit(s);
769
770     MPV_common_end(s);
771     if ((CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER) && s->out_format == FMT_MJPEG)
772         ff_mjpeg_encode_close(s);
773
774     av_freep(&avctx->extradata);
775
776     return 0;
777 }
778
779 static int get_sae(uint8_t *src, int ref, int stride){
780     int x,y;
781     int acc=0;
782
783     for(y=0; y<16; y++){
784         for(x=0; x<16; x++){
785             acc+= FFABS(src[x+y*stride] - ref);
786         }
787     }
788
789     return acc;
790 }
791
792 static int get_intra_count(MpegEncContext *s, uint8_t *src, uint8_t *ref, int stride){
793     int x, y, w, h;
794     int acc=0;
795
796     w= s->width &~15;
797     h= s->height&~15;
798
799     for(y=0; y<h; y+=16){
800         for(x=0; x<w; x+=16){
801             int offset= x + y*stride;
802             int sad = s->dsp.sad[0](NULL, src + offset, ref + offset, stride, 16);
803             int mean= (s->dsp.pix_sum(src + offset, stride) + 128)>>8;
804             int sae = get_sae(src + offset, mean, stride);
805
806             acc+= sae + 500 < sad;
807         }
808     }
809     return acc;
810 }
811
812
813 static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg){
814     AVFrame *pic=NULL;
815     int64_t pts;
816     int i;
817     const int encoding_delay= s->max_b_frames;
818     int direct=1;
819
820     if(pic_arg){
821         pts= pic_arg->pts;
822         pic_arg->display_picture_number= s->input_picture_number++;
823
824         if(pts != AV_NOPTS_VALUE){
825             if(s->user_specified_pts != AV_NOPTS_VALUE){
826                 int64_t time= pts;
827                 int64_t last= s->user_specified_pts;
828
829                 if(time <= last){
830                     av_log(s->avctx, AV_LOG_ERROR, "Error, Invalid timestamp=%"PRId64", last=%"PRId64"\n", pts, s->user_specified_pts);
831                     return -1;
832                 }
833             }
834             s->user_specified_pts= pts;
835         }else{
836             if(s->user_specified_pts != AV_NOPTS_VALUE){
837                 s->user_specified_pts=
838                 pts= s->user_specified_pts + 1;
839                 av_log(s->avctx, AV_LOG_INFO, "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n", pts);
840             }else{
841                 pts= pic_arg->display_picture_number;
842             }
843         }
844     }
845
846   if(pic_arg){
847     if(encoding_delay && !(s->flags&CODEC_FLAG_INPUT_PRESERVED)) direct=0;
848     if(pic_arg->linesize[0] != s->linesize) direct=0;
849     if(pic_arg->linesize[1] != s->uvlinesize) direct=0;
850     if(pic_arg->linesize[2] != s->uvlinesize) direct=0;
851
852 //    av_log(AV_LOG_DEBUG, "%d %d %d %d\n",pic_arg->linesize[0], pic_arg->linesize[1], s->linesize, s->uvlinesize);
853
854     if(direct){
855         i= ff_find_unused_picture(s, 1);
856
857         pic= (AVFrame*)&s->picture[i];
858         pic->reference= 3;
859
860         for(i=0; i<4; i++){
861             pic->data[i]= pic_arg->data[i];
862             pic->linesize[i]= pic_arg->linesize[i];
863         }
864         if(ff_alloc_picture(s, (Picture*)pic, 1) < 0){
865             return -1;
866         }
867     }else{
868         i= ff_find_unused_picture(s, 0);
869
870         pic= (AVFrame*)&s->picture[i];
871         pic->reference= 3;
872
873         if(ff_alloc_picture(s, (Picture*)pic, 0) < 0){
874             return -1;
875         }
876
877         if(   pic->data[0] + INPLACE_OFFSET == pic_arg->data[0]
878            && pic->data[1] + INPLACE_OFFSET == pic_arg->data[1]
879            && pic->data[2] + INPLACE_OFFSET == pic_arg->data[2]){
880        // empty
881         }else{
882             int h_chroma_shift, v_chroma_shift;
883             avcodec_get_chroma_sub_sample(s->avctx->pix_fmt, &h_chroma_shift, &v_chroma_shift);
884
885             for(i=0; i<3; i++){
886                 int src_stride= pic_arg->linesize[i];
887                 int dst_stride= i ? s->uvlinesize : s->linesize;
888                 int h_shift= i ? h_chroma_shift : 0;
889                 int v_shift= i ? v_chroma_shift : 0;
890                 int w= s->width >>h_shift;
891                 int h= s->height>>v_shift;
892                 uint8_t *src= pic_arg->data[i];
893                 uint8_t *dst= pic->data[i];
894
895                 if(!s->avctx->rc_buffer_size)
896                     dst +=INPLACE_OFFSET;
897
898                 if(src_stride==dst_stride)
899                     memcpy(dst, src, src_stride*h);
900                 else{
901                     while(h--){
902                         memcpy(dst, src, w);
903                         dst += dst_stride;
904                         src += src_stride;
905                     }
906                 }
907             }
908         }
909     }
910     copy_picture_attributes(s, pic, pic_arg);
911     pic->pts= pts; //we set this here to avoid modifiying pic_arg
912   }
913
914     /* shift buffer entries */
915     for(i=1; i<MAX_PICTURE_COUNT /*s->encoding_delay+1*/; i++)
916         s->input_picture[i-1]= s->input_picture[i];
917
918     s->input_picture[encoding_delay]= (Picture*)pic;
919
920     return 0;
921 }
922
923 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref){
924     int x, y, plane;
925     int score=0;
926     int64_t score64=0;
927
928     for(plane=0; plane<3; plane++){
929         const int stride= p->linesize[plane];
930         const int bw= plane ? 1 : 2;
931         for(y=0; y<s->mb_height*bw; y++){
932             for(x=0; x<s->mb_width*bw; x++){
933                 int off= p->type == FF_BUFFER_TYPE_SHARED ? 0: 16;
934                 int v= s->dsp.frame_skip_cmp[1](s, p->data[plane] + 8*(x + y*stride)+off, ref->data[plane] + 8*(x + y*stride), stride, 8);
935
936                 switch(s->avctx->frame_skip_exp){
937                     case 0: score= FFMAX(score, v); break;
938                     case 1: score+= FFABS(v);break;
939                     case 2: score+= v*v;break;
940                     case 3: score64+= FFABS(v*v*(int64_t)v);break;
941                     case 4: score64+= v*v*(int64_t)(v*v);break;
942                 }
943             }
944         }
945     }
946
947     if(score) score64= score;
948
949     if(score64 < s->avctx->frame_skip_threshold)
950         return 1;
951     if(score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda)>>8))
952         return 1;
953     return 0;
954 }
955
956 static int estimate_best_b_count(MpegEncContext *s){
957     AVCodec *codec= avcodec_find_encoder(s->avctx->codec_id);
958     AVCodecContext *c= avcodec_alloc_context();
959     AVFrame input[FF_MAX_B_FRAMES+2];
960     const int scale= s->avctx->brd_scale;
961     int i, j, out_size, p_lambda, b_lambda, lambda2;
962     int outbuf_size= s->width * s->height; //FIXME
963     uint8_t *outbuf= av_malloc(outbuf_size);
964     int64_t best_rd= INT64_MAX;
965     int best_b_count= -1;
966
967     assert(scale>=0 && scale <=3);
968
969 //    emms_c();
970     p_lambda= s->last_lambda_for[AV_PICTURE_TYPE_P]; //s->next_picture_ptr->quality;
971     b_lambda= s->last_lambda_for[AV_PICTURE_TYPE_B]; //p_lambda *FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
972     if(!b_lambda) b_lambda= p_lambda; //FIXME we should do this somewhere else
973     lambda2= (b_lambda*b_lambda + (1<<FF_LAMBDA_SHIFT)/2 ) >> FF_LAMBDA_SHIFT;
974
975     c->width = s->width >> scale;
976     c->height= s->height>> scale;
977     c->flags= CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR | CODEC_FLAG_INPUT_PRESERVED /*| CODEC_FLAG_EMU_EDGE*/;
978     c->flags|= s->avctx->flags & CODEC_FLAG_QPEL;
979     c->mb_decision= s->avctx->mb_decision;
980     c->me_cmp= s->avctx->me_cmp;
981     c->mb_cmp= s->avctx->mb_cmp;
982     c->me_sub_cmp= s->avctx->me_sub_cmp;
983     c->pix_fmt = PIX_FMT_YUV420P;
984     c->time_base= s->avctx->time_base;
985     c->max_b_frames= s->max_b_frames;
986
987     if (avcodec_open(c, codec) < 0)
988         return -1;
989
990     for(i=0; i<s->max_b_frames+2; i++){
991         int ysize= c->width*c->height;
992         int csize= (c->width/2)*(c->height/2);
993         Picture pre_input, *pre_input_ptr= i ? s->input_picture[i-1] : s->next_picture_ptr;
994
995         avcodec_get_frame_defaults(&input[i]);
996         input[i].data[0]= av_malloc(ysize + 2*csize);
997         input[i].data[1]= input[i].data[0] + ysize;
998         input[i].data[2]= input[i].data[1] + csize;
999         input[i].linesize[0]= c->width;
1000         input[i].linesize[1]=
1001         input[i].linesize[2]= c->width/2;
1002
1003         if(pre_input_ptr && (!i || s->input_picture[i-1])) {
1004             pre_input= *pre_input_ptr;
1005
1006             if(pre_input.type != FF_BUFFER_TYPE_SHARED && i) {
1007                 pre_input.data[0]+=INPLACE_OFFSET;
1008                 pre_input.data[1]+=INPLACE_OFFSET;
1009                 pre_input.data[2]+=INPLACE_OFFSET;
1010             }
1011
1012             s->dsp.shrink[scale](input[i].data[0], input[i].linesize[0], pre_input.data[0], pre_input.linesize[0], c->width, c->height);
1013             s->dsp.shrink[scale](input[i].data[1], input[i].linesize[1], pre_input.data[1], pre_input.linesize[1], c->width>>1, c->height>>1);
1014             s->dsp.shrink[scale](input[i].data[2], input[i].linesize[2], pre_input.data[2], pre_input.linesize[2], c->width>>1, c->height>>1);
1015         }
1016     }
1017
1018     for(j=0; j<s->max_b_frames+1; j++){
1019         int64_t rd=0;
1020
1021         if(!s->input_picture[j])
1022             break;
1023
1024         c->error[0]= c->error[1]= c->error[2]= 0;
1025
1026         input[0].pict_type= AV_PICTURE_TYPE_I;
1027         input[0].quality= 1 * FF_QP2LAMBDA;
1028         out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[0]);
1029 //        rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1030
1031         for(i=0; i<s->max_b_frames+1; i++){
1032             int is_p= i % (j+1) == j || i==s->max_b_frames;
1033
1034             input[i+1].pict_type= is_p ? AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1035             input[i+1].quality= is_p ? p_lambda : b_lambda;
1036             out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[i+1]);
1037             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1038         }
1039
1040         /* get the delayed frames */
1041         while(out_size){
1042             out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL);
1043             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1044         }
1045
1046         rd += c->error[0] + c->error[1] + c->error[2];
1047
1048         if(rd < best_rd){
1049             best_rd= rd;
1050             best_b_count= j;
1051         }
1052     }
1053
1054     av_freep(&outbuf);
1055     avcodec_close(c);
1056     av_freep(&c);
1057
1058     for(i=0; i<s->max_b_frames+2; i++){
1059         av_freep(&input[i].data[0]);
1060     }
1061
1062     return best_b_count;
1063 }
1064
1065 static int select_input_picture(MpegEncContext *s){
1066     int i;
1067
1068     for(i=1; i<MAX_PICTURE_COUNT; i++)
1069         s->reordered_input_picture[i-1]= s->reordered_input_picture[i];
1070     s->reordered_input_picture[MAX_PICTURE_COUNT-1]= NULL;
1071
1072     /* set next picture type & ordering */
1073     if(s->reordered_input_picture[0]==NULL && s->input_picture[0]){
1074         if(/*s->picture_in_gop_number >= s->gop_size ||*/ s->next_picture_ptr==NULL || s->intra_only){
1075             s->reordered_input_picture[0]= s->input_picture[0];
1076             s->reordered_input_picture[0]->pict_type= AV_PICTURE_TYPE_I;
1077             s->reordered_input_picture[0]->coded_picture_number= s->coded_picture_number++;
1078         }else{
1079             int b_frames;
1080
1081             if(s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor){
1082                 if(s->picture_in_gop_number < s->gop_size && skip_check(s, s->input_picture[0], s->next_picture_ptr)){
1083                 //FIXME check that te gop check above is +-1 correct
1084 //av_log(NULL, AV_LOG_DEBUG, "skip %p %"PRId64"\n", s->input_picture[0]->data[0], s->input_picture[0]->pts);
1085
1086                     if(s->input_picture[0]->type == FF_BUFFER_TYPE_SHARED){
1087                         for(i=0; i<4; i++)
1088                             s->input_picture[0]->data[i]= NULL;
1089                         s->input_picture[0]->type= 0;
1090                     }else{
1091                         assert(   s->input_picture[0]->type==FF_BUFFER_TYPE_USER
1092                                || s->input_picture[0]->type==FF_BUFFER_TYPE_INTERNAL);
1093
1094                         s->avctx->release_buffer(s->avctx, (AVFrame*)s->input_picture[0]);
1095                     }
1096
1097                     emms_c();
1098                     ff_vbv_update(s, 0);
1099
1100                     goto no_output_pic;
1101                 }
1102             }
1103
1104             if(s->flags&CODEC_FLAG_PASS2){
1105                 for(i=0; i<s->max_b_frames+1; i++){
1106                     int pict_num= s->input_picture[0]->display_picture_number + i;
1107
1108                     if(pict_num >= s->rc_context.num_entries)
1109                         break;
1110                     if(!s->input_picture[i]){
1111                         s->rc_context.entry[pict_num-1].new_pict_type = AV_PICTURE_TYPE_P;
1112                         break;
1113                     }
1114
1115                     s->input_picture[i]->pict_type=
1116                         s->rc_context.entry[pict_num].new_pict_type;
1117                 }
1118             }
1119
1120             if(s->avctx->b_frame_strategy==0){
1121                 b_frames= s->max_b_frames;
1122                 while(b_frames && !s->input_picture[b_frames]) b_frames--;
1123             }else if(s->avctx->b_frame_strategy==1){
1124                 for(i=1; i<s->max_b_frames+1; i++){
1125                     if(s->input_picture[i] && s->input_picture[i]->b_frame_score==0){
1126                         s->input_picture[i]->b_frame_score=
1127                             get_intra_count(s, s->input_picture[i  ]->data[0],
1128                                                s->input_picture[i-1]->data[0], s->linesize) + 1;
1129                     }
1130                 }
1131                 for(i=0; i<s->max_b_frames+1; i++){
1132                     if(s->input_picture[i]==NULL || s->input_picture[i]->b_frame_score - 1 > s->mb_num/s->avctx->b_sensitivity) break;
1133                 }
1134
1135                 b_frames= FFMAX(0, i-1);
1136
1137                 /* reset scores */
1138                 for(i=0; i<b_frames+1; i++){
1139                     s->input_picture[i]->b_frame_score=0;
1140                 }
1141             }else if(s->avctx->b_frame_strategy==2){
1142                 b_frames= estimate_best_b_count(s);
1143             }else{
1144                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1145                 b_frames=0;
1146             }
1147
1148             emms_c();
1149 //static int b_count=0;
1150 //b_count+= b_frames;
1151 //av_log(s->avctx, AV_LOG_DEBUG, "b_frames: %d\n", b_count);
1152
1153             for(i= b_frames - 1; i>=0; i--){
1154                 int type= s->input_picture[i]->pict_type;
1155                 if(type && type != AV_PICTURE_TYPE_B)
1156                     b_frames= i;
1157             }
1158             if(s->input_picture[b_frames]->pict_type == AV_PICTURE_TYPE_B && b_frames == s->max_b_frames){
1159                 av_log(s->avctx, AV_LOG_ERROR, "warning, too many b frames in a row\n");
1160             }
1161
1162             if(s->picture_in_gop_number + b_frames >= s->gop_size){
1163               if((s->flags2 & CODEC_FLAG2_STRICT_GOP) && s->gop_size > s->picture_in_gop_number){
1164                     b_frames= s->gop_size - s->picture_in_gop_number - 1;
1165               }else{
1166                 if(s->flags & CODEC_FLAG_CLOSED_GOP)
1167                     b_frames=0;
1168                 s->input_picture[b_frames]->pict_type= AV_PICTURE_TYPE_I;
1169               }
1170             }
1171
1172             if(   (s->flags & CODEC_FLAG_CLOSED_GOP)
1173                && b_frames
1174                && s->input_picture[b_frames]->pict_type== AV_PICTURE_TYPE_I)
1175                 b_frames--;
1176
1177             s->reordered_input_picture[0]= s->input_picture[b_frames];
1178             if(s->reordered_input_picture[0]->pict_type != AV_PICTURE_TYPE_I)
1179                 s->reordered_input_picture[0]->pict_type= AV_PICTURE_TYPE_P;
1180             s->reordered_input_picture[0]->coded_picture_number= s->coded_picture_number++;
1181             for(i=0; i<b_frames; i++){
1182                 s->reordered_input_picture[i+1]= s->input_picture[i];
1183                 s->reordered_input_picture[i+1]->pict_type= AV_PICTURE_TYPE_B;
1184                 s->reordered_input_picture[i+1]->coded_picture_number= s->coded_picture_number++;
1185             }
1186         }
1187     }
1188 no_output_pic:
1189     if(s->reordered_input_picture[0]){
1190         s->reordered_input_picture[0]->reference= s->reordered_input_picture[0]->pict_type!=AV_PICTURE_TYPE_B ? 3 : 0;
1191
1192         ff_copy_picture(&s->new_picture, s->reordered_input_picture[0]);
1193
1194         if(s->reordered_input_picture[0]->type == FF_BUFFER_TYPE_SHARED || s->avctx->rc_buffer_size){
1195             // input is a shared pix, so we can't modifiy it -> alloc a new one & ensure that the shared one is reuseable
1196
1197             int i= ff_find_unused_picture(s, 0);
1198             Picture *pic= &s->picture[i];
1199
1200             pic->reference              = s->reordered_input_picture[0]->reference;
1201             if(ff_alloc_picture(s, pic, 0) < 0){
1202                 return -1;
1203             }
1204
1205             /* mark us unused / free shared pic */
1206             if(s->reordered_input_picture[0]->type == FF_BUFFER_TYPE_INTERNAL)
1207                 s->avctx->release_buffer(s->avctx, (AVFrame*)s->reordered_input_picture[0]);
1208             for(i=0; i<4; i++)
1209                 s->reordered_input_picture[0]->data[i]= NULL;
1210             s->reordered_input_picture[0]->type= 0;
1211
1212             copy_picture_attributes(s, (AVFrame*)pic, (AVFrame*)s->reordered_input_picture[0]);
1213
1214             s->current_picture_ptr= pic;
1215         }else{
1216             // input is not a shared pix -> reuse buffer for current_pix
1217
1218             assert(   s->reordered_input_picture[0]->type==FF_BUFFER_TYPE_USER
1219                    || s->reordered_input_picture[0]->type==FF_BUFFER_TYPE_INTERNAL);
1220
1221             s->current_picture_ptr= s->reordered_input_picture[0];
1222             for(i=0; i<4; i++){
1223                 s->new_picture.data[i]+= INPLACE_OFFSET;
1224             }
1225         }
1226         ff_copy_picture(&s->current_picture, s->current_picture_ptr);
1227
1228         s->picture_number= s->new_picture.display_picture_number;
1229 //printf("dpn:%d\n", s->picture_number);
1230     }else{
1231        memset(&s->new_picture, 0, sizeof(Picture));
1232     }
1233     return 0;
1234 }
1235
1236 int MPV_encode_picture(AVCodecContext *avctx,
1237                        unsigned char *buf, int buf_size, void *data)
1238 {
1239     MpegEncContext *s = avctx->priv_data;
1240     AVFrame *pic_arg = data;
1241     int i, stuffing_count;
1242
1243     for(i=0; i<avctx->thread_count; i++){
1244         int start_y= s->thread_context[i]->start_mb_y;
1245         int   end_y= s->thread_context[i]->  end_mb_y;
1246         int h= s->mb_height;
1247         uint8_t *start= buf + (size_t)(((int64_t) buf_size)*start_y/h);
1248         uint8_t *end  = buf + (size_t)(((int64_t) buf_size)*  end_y/h);
1249
1250         init_put_bits(&s->thread_context[i]->pb, start, end - start);
1251     }
1252
1253     s->picture_in_gop_number++;
1254
1255     if(load_input_picture(s, pic_arg) < 0)
1256         return -1;
1257
1258     if(select_input_picture(s) < 0){
1259         return -1;
1260     }
1261
1262     /* output? */
1263     if(s->new_picture.data[0]){
1264         s->pict_type= s->new_picture.pict_type;
1265 //emms_c();
1266 //printf("qs:%f %f %d\n", s->new_picture.quality, s->current_picture.quality, s->qscale);
1267         MPV_frame_start(s, avctx);
1268 vbv_retry:
1269         if (encode_picture(s, s->picture_number) < 0)
1270             return -1;
1271
1272         avctx->header_bits = s->header_bits;
1273         avctx->mv_bits     = s->mv_bits;
1274         avctx->misc_bits   = s->misc_bits;
1275         avctx->i_tex_bits  = s->i_tex_bits;
1276         avctx->p_tex_bits  = s->p_tex_bits;
1277         avctx->i_count     = s->i_count;
1278         avctx->p_count     = s->mb_num - s->i_count - s->skip_count; //FIXME f/b_count in avctx
1279         avctx->skip_count  = s->skip_count;
1280
1281         MPV_frame_end(s);
1282
1283         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1284             ff_mjpeg_encode_picture_trailer(s);
1285
1286         if(avctx->rc_buffer_size){
1287             RateControlContext *rcc= &s->rc_context;
1288             int max_size= rcc->buffer_index * avctx->rc_max_available_vbv_use;
1289
1290             if(put_bits_count(&s->pb) > max_size && s->lambda < s->avctx->lmax){
1291                 s->next_lambda= FFMAX(s->lambda+1, s->lambda*(s->qscale+1) / s->qscale);
1292                 if(s->adaptive_quant){
1293                     int i;
1294                     for(i=0; i<s->mb_height*s->mb_stride; i++)
1295                         s->lambda_table[i]= FFMAX(s->lambda_table[i]+1, s->lambda_table[i]*(s->qscale+1) / s->qscale);
1296                 }
1297                 s->mb_skipped = 0;        //done in MPV_frame_start()
1298                 if(s->pict_type==AV_PICTURE_TYPE_P){ //done in encode_picture() so we must undo it
1299                     if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
1300                         s->no_rounding ^= 1;
1301                 }
1302                 if(s->pict_type!=AV_PICTURE_TYPE_B){
1303                     s->time_base= s->last_time_base;
1304                     s->last_non_b_time= s->time - s->pp_time;
1305                 }
1306 //                av_log(NULL, AV_LOG_ERROR, "R:%d ", s->next_lambda);
1307                 for(i=0; i<avctx->thread_count; i++){
1308                     PutBitContext *pb= &s->thread_context[i]->pb;
1309                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1310                 }
1311                 goto vbv_retry;
1312             }
1313
1314             assert(s->avctx->rc_max_rate);
1315         }
1316
1317         if(s->flags&CODEC_FLAG_PASS1)
1318             ff_write_pass1_stats(s);
1319
1320         for(i=0; i<4; i++){
1321             s->current_picture_ptr->error[i]= s->current_picture.error[i];
1322             avctx->error[i] += s->current_picture_ptr->error[i];
1323         }
1324
1325         if(s->flags&CODEC_FLAG_PASS1)
1326             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits + avctx->i_tex_bits + avctx->p_tex_bits == put_bits_count(&s->pb));
1327         flush_put_bits(&s->pb);
1328         s->frame_bits  = put_bits_count(&s->pb);
1329
1330         stuffing_count= ff_vbv_update(s, s->frame_bits);
1331         if(stuffing_count){
1332             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < stuffing_count + 50){
1333                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1334                 return -1;
1335             }
1336
1337             switch(s->codec_id){
1338             case CODEC_ID_MPEG1VIDEO:
1339             case CODEC_ID_MPEG2VIDEO:
1340                 while(stuffing_count--){
1341                     put_bits(&s->pb, 8, 0);
1342                 }
1343             break;
1344             case CODEC_ID_MPEG4:
1345                 put_bits(&s->pb, 16, 0);
1346                 put_bits(&s->pb, 16, 0x1C3);
1347                 stuffing_count -= 4;
1348                 while(stuffing_count--){
1349                     put_bits(&s->pb, 8, 0xFF);
1350                 }
1351             break;
1352             default:
1353                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1354             }
1355             flush_put_bits(&s->pb);
1356             s->frame_bits  = put_bits_count(&s->pb);
1357         }
1358
1359         /* update mpeg1/2 vbv_delay for CBR */
1360         if(s->avctx->rc_max_rate && s->avctx->rc_min_rate == s->avctx->rc_max_rate && s->out_format == FMT_MPEG1
1361            && 90000LL * (avctx->rc_buffer_size-1) <= s->avctx->rc_max_rate*0xFFFFLL){
1362             int vbv_delay, min_delay;
1363             double inbits = s->avctx->rc_max_rate*av_q2d(s->avctx->time_base);
1364             int    minbits= s->frame_bits - 8*(s->vbv_delay_ptr - s->pb.buf - 1);
1365             double bits   = s->rc_context.buffer_index + minbits - inbits;
1366
1367             if(bits<0)
1368                 av_log(s->avctx, AV_LOG_ERROR, "Internal error, negative bits\n");
1369
1370             assert(s->repeat_first_field==0);
1371
1372             vbv_delay=     bits * 90000                               / s->avctx->rc_max_rate;
1373             min_delay= (minbits * 90000LL + s->avctx->rc_max_rate - 1)/ s->avctx->rc_max_rate;
1374
1375             vbv_delay= FFMAX(vbv_delay, min_delay);
1376
1377             assert(vbv_delay < 0xFFFF);
1378
1379             s->vbv_delay_ptr[0] &= 0xF8;
1380             s->vbv_delay_ptr[0] |= vbv_delay>>13;
1381             s->vbv_delay_ptr[1]  = vbv_delay>>5;
1382             s->vbv_delay_ptr[2] &= 0x07;
1383             s->vbv_delay_ptr[2] |= vbv_delay<<3;
1384             avctx->vbv_delay = vbv_delay*300;
1385         }
1386         s->total_bits += s->frame_bits;
1387         avctx->frame_bits  = s->frame_bits;
1388     }else{
1389         assert((put_bits_ptr(&s->pb) == s->pb.buf));
1390         s->frame_bits=0;
1391     }
1392     assert((s->frame_bits&7)==0);
1393
1394     return s->frame_bits/8;
1395 }
1396
1397 static inline void dct_single_coeff_elimination(MpegEncContext *s, int n, int threshold)
1398 {
1399     static const char tab[64]=
1400         {3,2,2,1,1,1,1,1,
1401          1,1,1,1,1,1,1,1,
1402          1,1,1,1,1,1,1,1,
1403          0,0,0,0,0,0,0,0,
1404          0,0,0,0,0,0,0,0,
1405          0,0,0,0,0,0,0,0,
1406          0,0,0,0,0,0,0,0,
1407          0,0,0,0,0,0,0,0};
1408     int score=0;
1409     int run=0;
1410     int i;
1411     DCTELEM *block= s->block[n];
1412     const int last_index= s->block_last_index[n];
1413     int skip_dc;
1414
1415     if(threshold<0){
1416         skip_dc=0;
1417         threshold= -threshold;
1418     }else
1419         skip_dc=1;
1420
1421     /* Are all we could set to zero already zero? */
1422     if(last_index<=skip_dc - 1) return;
1423
1424     for(i=0; i<=last_index; i++){
1425         const int j = s->intra_scantable.permutated[i];
1426         const int level = FFABS(block[j]);
1427         if(level==1){
1428             if(skip_dc && i==0) continue;
1429             score+= tab[run];
1430             run=0;
1431         }else if(level>1){
1432             return;
1433         }else{
1434             run++;
1435         }
1436     }
1437     if(score >= threshold) return;
1438     for(i=skip_dc; i<=last_index; i++){
1439         const int j = s->intra_scantable.permutated[i];
1440         block[j]=0;
1441     }
1442     if(block[0]) s->block_last_index[n]= 0;
1443     else         s->block_last_index[n]= -1;
1444 }
1445
1446 static inline void clip_coeffs(MpegEncContext *s, DCTELEM *block, int last_index)
1447 {
1448     int i;
1449     const int maxlevel= s->max_qcoeff;
1450     const int minlevel= s->min_qcoeff;
1451     int overflow=0;
1452
1453     if(s->mb_intra){
1454         i=1; //skip clipping of intra dc
1455     }else
1456         i=0;
1457
1458     for(;i<=last_index; i++){
1459         const int j= s->intra_scantable.permutated[i];
1460         int level = block[j];
1461
1462         if     (level>maxlevel){
1463             level=maxlevel;
1464             overflow++;
1465         }else if(level<minlevel){
1466             level=minlevel;
1467             overflow++;
1468         }
1469
1470         block[j]= level;
1471     }
1472
1473     if(overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1474         av_log(s->avctx, AV_LOG_INFO, "warning, clipping %d dct coefficients to %d..%d\n", overflow, minlevel, maxlevel);
1475 }
1476
1477 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride){
1478     int x, y;
1479 //FIXME optimize
1480     for(y=0; y<8; y++){
1481         for(x=0; x<8; x++){
1482             int x2, y2;
1483             int sum=0;
1484             int sqr=0;
1485             int count=0;
1486
1487             for(y2= FFMAX(y-1, 0); y2 < FFMIN(8, y+2); y2++){
1488                 for(x2= FFMAX(x-1, 0); x2 < FFMIN(8, x+2); x2++){
1489                     int v= ptr[x2 + y2*stride];
1490                     sum += v;
1491                     sqr += v*v;
1492                     count++;
1493                 }
1494             }
1495             weight[x + 8*y]= (36*ff_sqrt(count*sqr - sum*sum)) / count;
1496         }
1497     }
1498 }
1499
1500 static av_always_inline void encode_mb_internal(MpegEncContext *s, int motion_x, int motion_y, int mb_block_height, int mb_block_count)
1501 {
1502     int16_t weight[8][64];
1503     DCTELEM orig[8][64];
1504     const int mb_x= s->mb_x;
1505     const int mb_y= s->mb_y;
1506     int i;
1507     int skip_dct[8];
1508     int dct_offset   = s->linesize*8; //default for progressive frames
1509     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1510     int wrap_y, wrap_c;
1511
1512     for(i=0; i<mb_block_count; i++) skip_dct[i]=s->skipdct;
1513
1514     if(s->adaptive_quant){
1515         const int last_qp= s->qscale;
1516         const int mb_xy= mb_x + mb_y*s->mb_stride;
1517
1518         s->lambda= s->lambda_table[mb_xy];
1519         update_qscale(s);
1520
1521         if(!(s->flags&CODEC_FLAG_QP_RD)){
1522             s->qscale= s->current_picture_ptr->qscale_table[mb_xy];
1523             s->dquant= s->qscale - last_qp;
1524
1525             if(s->out_format==FMT_H263){
1526                 s->dquant= av_clip(s->dquant, -2, 2);
1527
1528                 if(s->codec_id==CODEC_ID_MPEG4){
1529                     if(!s->mb_intra){
1530                         if(s->pict_type == AV_PICTURE_TYPE_B){
1531                             if(s->dquant&1 || s->mv_dir&MV_DIRECT)
1532                                 s->dquant= 0;
1533                         }
1534                         if(s->mv_type==MV_TYPE_8X8)
1535                             s->dquant=0;
1536                     }
1537                 }
1538             }
1539         }
1540         ff_set_qscale(s, last_qp + s->dquant);
1541     }else if(s->flags&CODEC_FLAG_QP_RD)
1542         ff_set_qscale(s, s->qscale + s->dquant);
1543
1544     wrap_y = s->linesize;
1545     wrap_c = s->uvlinesize;
1546     ptr_y = s->new_picture.data[0] + (mb_y * 16 * wrap_y) + mb_x * 16;
1547     ptr_cb = s->new_picture.data[1] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1548     ptr_cr = s->new_picture.data[2] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1549
1550     if(mb_x*16+16 > s->width || mb_y*16+16 > s->height){
1551         uint8_t *ebuf= s->edge_emu_buffer + 32;
1552         s->dsp.emulated_edge_mc(ebuf            , ptr_y , wrap_y,16,16,mb_x*16,mb_y*16, s->width   , s->height);
1553         ptr_y= ebuf;
1554         s->dsp.emulated_edge_mc(ebuf+18*wrap_y  , ptr_cb, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1555         ptr_cb= ebuf+18*wrap_y;
1556         s->dsp.emulated_edge_mc(ebuf+18*wrap_y+8, ptr_cr, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1557         ptr_cr= ebuf+18*wrap_y+8;
1558     }
1559
1560     if (s->mb_intra) {
1561         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1562             int progressive_score, interlaced_score;
1563
1564             s->interlaced_dct=0;
1565             progressive_score= s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y, 8)
1566                               +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y*8, NULL, wrap_y, 8) - 400;
1567
1568             if(progressive_score > 0){
1569                 interlaced_score = s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y*2, 8)
1570                                   +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y  , NULL, wrap_y*2, 8);
1571                 if(progressive_score > interlaced_score){
1572                     s->interlaced_dct=1;
1573
1574                     dct_offset= wrap_y;
1575                     wrap_y<<=1;
1576                     if (s->chroma_format == CHROMA_422)
1577                         wrap_c<<=1;
1578                 }
1579             }
1580         }
1581
1582         s->dsp.get_pixels(s->block[0], ptr_y                 , wrap_y);
1583         s->dsp.get_pixels(s->block[1], ptr_y              + 8, wrap_y);
1584         s->dsp.get_pixels(s->block[2], ptr_y + dct_offset    , wrap_y);
1585         s->dsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
1586
1587         if(s->flags&CODEC_FLAG_GRAY){
1588             skip_dct[4]= 1;
1589             skip_dct[5]= 1;
1590         }else{
1591             s->dsp.get_pixels(s->block[4], ptr_cb, wrap_c);
1592             s->dsp.get_pixels(s->block[5], ptr_cr, wrap_c);
1593             if(!s->chroma_y_shift){ /* 422 */
1594                 s->dsp.get_pixels(s->block[6], ptr_cb + (dct_offset>>1), wrap_c);
1595                 s->dsp.get_pixels(s->block[7], ptr_cr + (dct_offset>>1), wrap_c);
1596             }
1597         }
1598     }else{
1599         op_pixels_func (*op_pix)[4];
1600         qpel_mc_func (*op_qpix)[16];
1601         uint8_t *dest_y, *dest_cb, *dest_cr;
1602
1603         dest_y  = s->dest[0];
1604         dest_cb = s->dest[1];
1605         dest_cr = s->dest[2];
1606
1607         if ((!s->no_rounding) || s->pict_type==AV_PICTURE_TYPE_B){
1608             op_pix = s->dsp.put_pixels_tab;
1609             op_qpix= s->dsp.put_qpel_pixels_tab;
1610         }else{
1611             op_pix = s->dsp.put_no_rnd_pixels_tab;
1612             op_qpix= s->dsp.put_no_rnd_qpel_pixels_tab;
1613         }
1614
1615         if (s->mv_dir & MV_DIR_FORWARD) {
1616             MPV_motion(s, dest_y, dest_cb, dest_cr, 0, s->last_picture.data, op_pix, op_qpix);
1617             op_pix = s->dsp.avg_pixels_tab;
1618             op_qpix= s->dsp.avg_qpel_pixels_tab;
1619         }
1620         if (s->mv_dir & MV_DIR_BACKWARD) {
1621             MPV_motion(s, dest_y, dest_cb, dest_cr, 1, s->next_picture.data, op_pix, op_qpix);
1622         }
1623
1624         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1625             int progressive_score, interlaced_score;
1626
1627             s->interlaced_dct=0;
1628             progressive_score= s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y, 8)
1629                               +s->dsp.ildct_cmp[0](s, dest_y + wrap_y*8, ptr_y + wrap_y*8, wrap_y, 8) - 400;
1630
1631             if(s->avctx->ildct_cmp == FF_CMP_VSSE) progressive_score -= 400;
1632
1633             if(progressive_score>0){
1634                 interlaced_score = s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y*2, 8)
1635                                   +s->dsp.ildct_cmp[0](s, dest_y + wrap_y  , ptr_y + wrap_y  , wrap_y*2, 8);
1636
1637                 if(progressive_score > interlaced_score){
1638                     s->interlaced_dct=1;
1639
1640                     dct_offset= wrap_y;
1641                     wrap_y<<=1;
1642                     if (s->chroma_format == CHROMA_422)
1643                         wrap_c<<=1;
1644                 }
1645             }
1646         }
1647
1648         s->dsp.diff_pixels(s->block[0], ptr_y                 , dest_y                 , wrap_y);
1649         s->dsp.diff_pixels(s->block[1], ptr_y              + 8, dest_y              + 8, wrap_y);
1650         s->dsp.diff_pixels(s->block[2], ptr_y + dct_offset    , dest_y + dct_offset    , wrap_y);
1651         s->dsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8, dest_y + dct_offset + 8, wrap_y);
1652
1653         if(s->flags&CODEC_FLAG_GRAY){
1654             skip_dct[4]= 1;
1655             skip_dct[5]= 1;
1656         }else{
1657             s->dsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
1658             s->dsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
1659             if(!s->chroma_y_shift){ /* 422 */
1660                 s->dsp.diff_pixels(s->block[6], ptr_cb + (dct_offset>>1), dest_cb + (dct_offset>>1), wrap_c);
1661                 s->dsp.diff_pixels(s->block[7], ptr_cr + (dct_offset>>1), dest_cr + (dct_offset>>1), wrap_c);
1662             }
1663         }
1664         /* pre quantization */
1665         if(s->current_picture.mc_mb_var[s->mb_stride*mb_y+ mb_x]<2*s->qscale*s->qscale){
1666             //FIXME optimize
1667             if(s->dsp.sad[1](NULL, ptr_y               , dest_y               , wrap_y, 8) < 20*s->qscale) skip_dct[0]= 1;
1668             if(s->dsp.sad[1](NULL, ptr_y            + 8, dest_y            + 8, wrap_y, 8) < 20*s->qscale) skip_dct[1]= 1;
1669             if(s->dsp.sad[1](NULL, ptr_y +dct_offset   , dest_y +dct_offset   , wrap_y, 8) < 20*s->qscale) skip_dct[2]= 1;
1670             if(s->dsp.sad[1](NULL, ptr_y +dct_offset+ 8, dest_y +dct_offset+ 8, wrap_y, 8) < 20*s->qscale) skip_dct[3]= 1;
1671             if(s->dsp.sad[1](NULL, ptr_cb              , dest_cb              , wrap_c, 8) < 20*s->qscale) skip_dct[4]= 1;
1672             if(s->dsp.sad[1](NULL, ptr_cr              , dest_cr              , wrap_c, 8) < 20*s->qscale) skip_dct[5]= 1;
1673             if(!s->chroma_y_shift){ /* 422 */
1674                 if(s->dsp.sad[1](NULL, ptr_cb +(dct_offset>>1), dest_cb +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[6]= 1;
1675                 if(s->dsp.sad[1](NULL, ptr_cr +(dct_offset>>1), dest_cr +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[7]= 1;
1676             }
1677         }
1678     }
1679
1680     if(s->avctx->quantizer_noise_shaping){
1681         if(!skip_dct[0]) get_visual_weight(weight[0], ptr_y                 , wrap_y);
1682         if(!skip_dct[1]) get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
1683         if(!skip_dct[2]) get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
1684         if(!skip_dct[3]) get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
1685         if(!skip_dct[4]) get_visual_weight(weight[4], ptr_cb                , wrap_c);
1686         if(!skip_dct[5]) get_visual_weight(weight[5], ptr_cr                , wrap_c);
1687         if(!s->chroma_y_shift){ /* 422 */
1688             if(!skip_dct[6]) get_visual_weight(weight[6], ptr_cb + (dct_offset>>1), wrap_c);
1689             if(!skip_dct[7]) get_visual_weight(weight[7], ptr_cr + (dct_offset>>1), wrap_c);
1690         }
1691         memcpy(orig[0], s->block[0], sizeof(DCTELEM)*64*mb_block_count);
1692     }
1693
1694     /* DCT & quantize */
1695     assert(s->out_format!=FMT_MJPEG || s->qscale==8);
1696     {
1697         for(i=0;i<mb_block_count;i++) {
1698             if(!skip_dct[i]){
1699                 int overflow;
1700                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
1701             // FIXME we could decide to change to quantizer instead of clipping
1702             // JS: I don't think that would be a good idea it could lower quality instead
1703             //     of improve it. Just INTRADC clipping deserves changes in quantizer
1704                 if (overflow) clip_coeffs(s, s->block[i], s->block_last_index[i]);
1705             }else
1706                 s->block_last_index[i]= -1;
1707         }
1708         if(s->avctx->quantizer_noise_shaping){
1709             for(i=0;i<mb_block_count;i++) {
1710                 if(!skip_dct[i]){
1711                     s->block_last_index[i] = dct_quantize_refine(s, s->block[i], weight[i], orig[i], i, s->qscale);
1712                 }
1713             }
1714         }
1715
1716         if(s->luma_elim_threshold && !s->mb_intra)
1717             for(i=0; i<4; i++)
1718                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
1719         if(s->chroma_elim_threshold && !s->mb_intra)
1720             for(i=4; i<mb_block_count; i++)
1721                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
1722
1723         if(s->flags & CODEC_FLAG_CBP_RD){
1724             for(i=0;i<mb_block_count;i++) {
1725                 if(s->block_last_index[i] == -1)
1726                     s->coded_score[i]= INT_MAX/256;
1727             }
1728         }
1729     }
1730
1731     if((s->flags&CODEC_FLAG_GRAY) && s->mb_intra){
1732         s->block_last_index[4]=
1733         s->block_last_index[5]= 0;
1734         s->block[4][0]=
1735         s->block[5][0]= (1024 + s->c_dc_scale/2)/ s->c_dc_scale;
1736     }
1737
1738     //non c quantize code returns incorrect block_last_index FIXME
1739     if(s->alternate_scan && s->dct_quantize != dct_quantize_c){
1740         for(i=0; i<mb_block_count; i++){
1741             int j;
1742             if(s->block_last_index[i]>0){
1743                 for(j=63; j>0; j--){
1744                     if(s->block[i][ s->intra_scantable.permutated[j] ]) break;
1745                 }
1746                 s->block_last_index[i]= j;
1747             }
1748         }
1749     }
1750
1751     /* huffman encode */
1752     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
1753     case CODEC_ID_MPEG1VIDEO:
1754     case CODEC_ID_MPEG2VIDEO:
1755         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
1756             mpeg1_encode_mb(s, s->block, motion_x, motion_y);
1757         break;
1758     case CODEC_ID_MPEG4:
1759         if (CONFIG_MPEG4_ENCODER)
1760             mpeg4_encode_mb(s, s->block, motion_x, motion_y);
1761         break;
1762     case CODEC_ID_MSMPEG4V2:
1763     case CODEC_ID_MSMPEG4V3:
1764     case CODEC_ID_WMV1:
1765         if (CONFIG_MSMPEG4_ENCODER)
1766             msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
1767         break;
1768     case CODEC_ID_WMV2:
1769         if (CONFIG_WMV2_ENCODER)
1770             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
1771         break;
1772     case CODEC_ID_H261:
1773         if (CONFIG_H261_ENCODER)
1774             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
1775         break;
1776     case CODEC_ID_H263:
1777     case CODEC_ID_H263P:
1778     case CODEC_ID_FLV1:
1779     case CODEC_ID_RV10:
1780     case CODEC_ID_RV20:
1781         if (CONFIG_H263_ENCODER)
1782             h263_encode_mb(s, s->block, motion_x, motion_y);
1783         break;
1784     case CODEC_ID_MJPEG:
1785         if (CONFIG_MJPEG_ENCODER)
1786             ff_mjpeg_encode_mb(s, s->block);
1787         break;
1788     default:
1789         assert(0);
1790     }
1791 }
1792
1793 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
1794 {
1795     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
1796     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
1797 }
1798
1799 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
1800     int i;
1801
1802     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster then a loop?
1803
1804     /* mpeg1 */
1805     d->mb_skip_run= s->mb_skip_run;
1806     for(i=0; i<3; i++)
1807         d->last_dc[i]= s->last_dc[i];
1808
1809     /* statistics */
1810     d->mv_bits= s->mv_bits;
1811     d->i_tex_bits= s->i_tex_bits;
1812     d->p_tex_bits= s->p_tex_bits;
1813     d->i_count= s->i_count;
1814     d->f_count= s->f_count;
1815     d->b_count= s->b_count;
1816     d->skip_count= s->skip_count;
1817     d->misc_bits= s->misc_bits;
1818     d->last_bits= 0;
1819
1820     d->mb_skipped= 0;
1821     d->qscale= s->qscale;
1822     d->dquant= s->dquant;
1823
1824     d->esc3_level_length= s->esc3_level_length;
1825 }
1826
1827 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
1828     int i;
1829
1830     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
1831     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster then a loop?
1832
1833     /* mpeg1 */
1834     d->mb_skip_run= s->mb_skip_run;
1835     for(i=0; i<3; i++)
1836         d->last_dc[i]= s->last_dc[i];
1837
1838     /* statistics */
1839     d->mv_bits= s->mv_bits;
1840     d->i_tex_bits= s->i_tex_bits;
1841     d->p_tex_bits= s->p_tex_bits;
1842     d->i_count= s->i_count;
1843     d->f_count= s->f_count;
1844     d->b_count= s->b_count;
1845     d->skip_count= s->skip_count;
1846     d->misc_bits= s->misc_bits;
1847
1848     d->mb_intra= s->mb_intra;
1849     d->mb_skipped= s->mb_skipped;
1850     d->mv_type= s->mv_type;
1851     d->mv_dir= s->mv_dir;
1852     d->pb= s->pb;
1853     if(s->data_partitioning){
1854         d->pb2= s->pb2;
1855         d->tex_pb= s->tex_pb;
1856     }
1857     d->block= s->block;
1858     for(i=0; i<8; i++)
1859         d->block_last_index[i]= s->block_last_index[i];
1860     d->interlaced_dct= s->interlaced_dct;
1861     d->qscale= s->qscale;
1862
1863     d->esc3_level_length= s->esc3_level_length;
1864 }
1865
1866 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
1867                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
1868                            int *dmin, int *next_block, int motion_x, int motion_y)
1869 {
1870     int score;
1871     uint8_t *dest_backup[3];
1872
1873     copy_context_before_encode(s, backup, type);
1874
1875     s->block= s->blocks[*next_block];
1876     s->pb= pb[*next_block];
1877     if(s->data_partitioning){
1878         s->pb2   = pb2   [*next_block];
1879         s->tex_pb= tex_pb[*next_block];
1880     }
1881
1882     if(*next_block){
1883         memcpy(dest_backup, s->dest, sizeof(s->dest));
1884         s->dest[0] = s->rd_scratchpad;
1885         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
1886         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
1887         assert(s->linesize >= 32); //FIXME
1888     }
1889
1890     encode_mb(s, motion_x, motion_y);
1891
1892     score= put_bits_count(&s->pb);
1893     if(s->data_partitioning){
1894         score+= put_bits_count(&s->pb2);
1895         score+= put_bits_count(&s->tex_pb);
1896     }
1897
1898     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
1899         MPV_decode_mb(s, s->block);
1900
1901         score *= s->lambda2;
1902         score += sse_mb(s) << FF_LAMBDA_SHIFT;
1903     }
1904
1905     if(*next_block){
1906         memcpy(s->dest, dest_backup, sizeof(s->dest));
1907     }
1908
1909     if(score<*dmin){
1910         *dmin= score;
1911         *next_block^=1;
1912
1913         copy_context_after_encode(best, s, type);
1914     }
1915 }
1916
1917 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
1918     uint32_t *sq = ff_squareTbl + 256;
1919     int acc=0;
1920     int x,y;
1921
1922     if(w==16 && h==16)
1923         return s->dsp.sse[0](NULL, src1, src2, stride, 16);
1924     else if(w==8 && h==8)
1925         return s->dsp.sse[1](NULL, src1, src2, stride, 8);
1926
1927     for(y=0; y<h; y++){
1928         for(x=0; x<w; x++){
1929             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
1930         }
1931     }
1932
1933     assert(acc>=0);
1934
1935     return acc;
1936 }
1937
1938 static int sse_mb(MpegEncContext *s){
1939     int w= 16;
1940     int h= 16;
1941
1942     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
1943     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
1944
1945     if(w==16 && h==16)
1946       if(s->avctx->mb_cmp == FF_CMP_NSSE){
1947         return  s->dsp.nsse[0](s, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
1948                +s->dsp.nsse[1](s, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
1949                +s->dsp.nsse[1](s, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
1950       }else{
1951         return  s->dsp.sse[0](NULL, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
1952                +s->dsp.sse[1](NULL, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
1953                +s->dsp.sse[1](NULL, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
1954       }
1955     else
1956         return  sse(s, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
1957                +sse(s, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
1958                +sse(s, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
1959 }
1960
1961 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
1962     MpegEncContext *s= *(void**)arg;
1963
1964
1965     s->me.pre_pass=1;
1966     s->me.dia_size= s->avctx->pre_dia_size;
1967     s->first_slice_line=1;
1968     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
1969         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
1970             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
1971         }
1972         s->first_slice_line=0;
1973     }
1974
1975     s->me.pre_pass=0;
1976
1977     return 0;
1978 }
1979
1980 static int estimate_motion_thread(AVCodecContext *c, void *arg){
1981     MpegEncContext *s= *(void**)arg;
1982
1983     ff_check_alignment();
1984
1985     s->me.dia_size= s->avctx->dia_size;
1986     s->first_slice_line=1;
1987     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
1988         s->mb_x=0; //for block init below
1989         ff_init_block_index(s);
1990         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
1991             s->block_index[0]+=2;
1992             s->block_index[1]+=2;
1993             s->block_index[2]+=2;
1994             s->block_index[3]+=2;
1995
1996             /* compute motion vector & mb_type and store in context */
1997             if(s->pict_type==AV_PICTURE_TYPE_B)
1998                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
1999             else
2000                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2001         }
2002         s->first_slice_line=0;
2003     }
2004     return 0;
2005 }
2006
2007 static int mb_var_thread(AVCodecContext *c, void *arg){
2008     MpegEncContext *s= *(void**)arg;
2009     int mb_x, mb_y;
2010
2011     ff_check_alignment();
2012
2013     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2014         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2015             int xx = mb_x * 16;
2016             int yy = mb_y * 16;
2017             uint8_t *pix = s->new_picture.data[0] + (yy * s->linesize) + xx;
2018             int varc;
2019             int sum = s->dsp.pix_sum(pix, s->linesize);
2020
2021             varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)(sum*sum))>>8) + 500 + 128)>>8;
2022
2023             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2024             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2025             s->me.mb_var_sum_temp    += varc;
2026         }
2027     }
2028     return 0;
2029 }
2030
2031 static void write_slice_end(MpegEncContext *s){
2032     if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4){
2033         if(s->partitioned_frame){
2034             ff_mpeg4_merge_partitions(s);
2035         }
2036
2037         ff_mpeg4_stuffing(&s->pb);
2038     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2039         ff_mjpeg_encode_stuffing(&s->pb);
2040     }
2041
2042     align_put_bits(&s->pb);
2043     flush_put_bits(&s->pb);
2044
2045     if((s->flags&CODEC_FLAG_PASS1) && !s->partitioned_frame)
2046         s->misc_bits+= get_bits_diff(s);
2047 }
2048
2049 static int encode_thread(AVCodecContext *c, void *arg){
2050     MpegEncContext *s= *(void**)arg;
2051     int mb_x, mb_y, pdif = 0;
2052     int chr_h= 16>>s->chroma_y_shift;
2053     int i, j;
2054     MpegEncContext best_s, backup_s;
2055     uint8_t bit_buf[2][MAX_MB_BYTES];
2056     uint8_t bit_buf2[2][MAX_MB_BYTES];
2057     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2058     PutBitContext pb[2], pb2[2], tex_pb[2];
2059 //printf("%d->%d\n", s->resync_mb_y, s->end_mb_y);
2060
2061     ff_check_alignment();
2062
2063     for(i=0; i<2; i++){
2064         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2065         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2066         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2067     }
2068
2069     s->last_bits= put_bits_count(&s->pb);
2070     s->mv_bits=0;
2071     s->misc_bits=0;
2072     s->i_tex_bits=0;
2073     s->p_tex_bits=0;
2074     s->i_count=0;
2075     s->f_count=0;
2076     s->b_count=0;
2077     s->skip_count=0;
2078
2079     for(i=0; i<3; i++){
2080         /* init last dc values */
2081         /* note: quant matrix value (8) is implied here */
2082         s->last_dc[i] = 128 << s->intra_dc_precision;
2083
2084         s->current_picture.error[i] = 0;
2085     }
2086     s->mb_skip_run = 0;
2087     memset(s->last_mv, 0, sizeof(s->last_mv));
2088
2089     s->last_mv_dir = 0;
2090
2091     switch(s->codec_id){
2092     case CODEC_ID_H263:
2093     case CODEC_ID_H263P:
2094     case CODEC_ID_FLV1:
2095         if (CONFIG_H263_ENCODER)
2096             s->gob_index = ff_h263_get_gob_height(s);
2097         break;
2098     case CODEC_ID_MPEG4:
2099         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2100             ff_mpeg4_init_partitions(s);
2101         break;
2102     }
2103
2104     s->resync_mb_x=0;
2105     s->resync_mb_y=0;
2106     s->first_slice_line = 1;
2107     s->ptr_lastgob = s->pb.buf;
2108     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2109 //    printf("row %d at %X\n", s->mb_y, (int)s);
2110         s->mb_x=0;
2111         s->mb_y= mb_y;
2112
2113         ff_set_qscale(s, s->qscale);
2114         ff_init_block_index(s);
2115
2116         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2117             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2118             int mb_type= s->mb_type[xy];
2119 //            int d;
2120             int dmin= INT_MAX;
2121             int dir;
2122
2123             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2124                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2125                 return -1;
2126             }
2127             if(s->data_partitioning){
2128                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2129                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2130                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2131                     return -1;
2132                 }
2133             }
2134
2135             s->mb_x = mb_x;
2136             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2137             ff_update_block_index(s);
2138
2139             if(CONFIG_H261_ENCODER && s->codec_id == CODEC_ID_H261){
2140                 ff_h261_reorder_mb_index(s);
2141                 xy= s->mb_y*s->mb_stride + s->mb_x;
2142                 mb_type= s->mb_type[xy];
2143             }
2144
2145             /* write gob / video packet header  */
2146             if(s->rtp_mode){
2147                 int current_packet_size, is_gob_start;
2148
2149                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2150
2151                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2152
2153                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2154
2155                 switch(s->codec_id){
2156                 case CODEC_ID_H263:
2157                 case CODEC_ID_H263P:
2158                     if(!s->h263_slice_structured)
2159                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2160                     break;
2161                 case CODEC_ID_MPEG2VIDEO:
2162                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2163                 case CODEC_ID_MPEG1VIDEO:
2164                     if(s->mb_skip_run) is_gob_start=0;
2165                     break;
2166                 }
2167
2168                 if(is_gob_start){
2169                     if(s->start_mb_y != mb_y || mb_x!=0){
2170                         write_slice_end(s);
2171
2172                         if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4 && s->partitioned_frame){
2173                             ff_mpeg4_init_partitions(s);
2174                         }
2175                     }
2176
2177                     assert((put_bits_count(&s->pb)&7) == 0);
2178                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2179
2180                     if(s->avctx->error_rate && s->resync_mb_x + s->resync_mb_y > 0){
2181                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2182                         int d= 100 / s->avctx->error_rate;
2183                         if(r % d == 0){
2184                             current_packet_size=0;
2185 #ifndef ALT_BITSTREAM_WRITER
2186                             s->pb.buf_ptr= s->ptr_lastgob;
2187 #endif
2188                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2189                         }
2190                     }
2191
2192                     if (s->avctx->rtp_callback){
2193                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2194                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2195                     }
2196
2197                     switch(s->codec_id){
2198                     case CODEC_ID_MPEG4:
2199                         if (CONFIG_MPEG4_ENCODER) {
2200                             ff_mpeg4_encode_video_packet_header(s);
2201                             ff_mpeg4_clean_buffers(s);
2202                         }
2203                     break;
2204                     case CODEC_ID_MPEG1VIDEO:
2205                     case CODEC_ID_MPEG2VIDEO:
2206                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2207                             ff_mpeg1_encode_slice_header(s);
2208                             ff_mpeg1_clean_buffers(s);
2209                         }
2210                     break;
2211                     case CODEC_ID_H263:
2212                     case CODEC_ID_H263P:
2213                         if (CONFIG_H263_ENCODER)
2214                             h263_encode_gob_header(s, mb_y);
2215                     break;
2216                     }
2217
2218                     if(s->flags&CODEC_FLAG_PASS1){
2219                         int bits= put_bits_count(&s->pb);
2220                         s->misc_bits+= bits - s->last_bits;
2221                         s->last_bits= bits;
2222                     }
2223
2224                     s->ptr_lastgob += current_packet_size;
2225                     s->first_slice_line=1;
2226                     s->resync_mb_x=mb_x;
2227                     s->resync_mb_y=mb_y;
2228                 }
2229             }
2230
2231             if(  (s->resync_mb_x   == s->mb_x)
2232                && s->resync_mb_y+1 == s->mb_y){
2233                 s->first_slice_line=0;
2234             }
2235
2236             s->mb_skipped=0;
2237             s->dquant=0; //only for QP_RD
2238
2239             if(mb_type & (mb_type-1) || (s->flags & CODEC_FLAG_QP_RD)){ // more than 1 MB type possible or CODEC_FLAG_QP_RD
2240                 int next_block=0;
2241                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2242
2243                 copy_context_before_encode(&backup_s, s, -1);
2244                 backup_s.pb= s->pb;
2245                 best_s.data_partitioning= s->data_partitioning;
2246                 best_s.partitioned_frame= s->partitioned_frame;
2247                 if(s->data_partitioning){
2248                     backup_s.pb2= s->pb2;
2249                     backup_s.tex_pb= s->tex_pb;
2250                 }
2251
2252                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2253                     s->mv_dir = MV_DIR_FORWARD;
2254                     s->mv_type = MV_TYPE_16X16;
2255                     s->mb_intra= 0;
2256                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2257                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2258                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2259                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2260                 }
2261                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2262                     s->mv_dir = MV_DIR_FORWARD;
2263                     s->mv_type = MV_TYPE_FIELD;
2264                     s->mb_intra= 0;
2265                     for(i=0; i<2; i++){
2266                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2267                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2268                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2269                     }
2270                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2271                                  &dmin, &next_block, 0, 0);
2272                 }
2273                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2274                     s->mv_dir = MV_DIR_FORWARD;
2275                     s->mv_type = MV_TYPE_16X16;
2276                     s->mb_intra= 0;
2277                     s->mv[0][0][0] = 0;
2278                     s->mv[0][0][1] = 0;
2279                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2280                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2281                 }
2282                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2283                     s->mv_dir = MV_DIR_FORWARD;
2284                     s->mv_type = MV_TYPE_8X8;
2285                     s->mb_intra= 0;
2286                     for(i=0; i<4; i++){
2287                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2288                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2289                     }
2290                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2291                                  &dmin, &next_block, 0, 0);
2292                 }
2293                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2294                     s->mv_dir = MV_DIR_FORWARD;
2295                     s->mv_type = MV_TYPE_16X16;
2296                     s->mb_intra= 0;
2297                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2298                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2299                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2300                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2301                 }
2302                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2303                     s->mv_dir = MV_DIR_BACKWARD;
2304                     s->mv_type = MV_TYPE_16X16;
2305                     s->mb_intra= 0;
2306                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2307                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2308                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2309                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2310                 }
2311                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2312                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2313                     s->mv_type = MV_TYPE_16X16;
2314                     s->mb_intra= 0;
2315                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2316                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2317                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2318                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2319                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2320                                  &dmin, &next_block, 0, 0);
2321                 }
2322                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2323                     s->mv_dir = MV_DIR_FORWARD;
2324                     s->mv_type = MV_TYPE_FIELD;
2325                     s->mb_intra= 0;
2326                     for(i=0; i<2; i++){
2327                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2328                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2329                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2330                     }
2331                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2332                                  &dmin, &next_block, 0, 0);
2333                 }
2334                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2335                     s->mv_dir = MV_DIR_BACKWARD;
2336                     s->mv_type = MV_TYPE_FIELD;
2337                     s->mb_intra= 0;
2338                     for(i=0; i<2; i++){
2339                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2340                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2341                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2342                     }
2343                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2344                                  &dmin, &next_block, 0, 0);
2345                 }
2346                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2347                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2348                     s->mv_type = MV_TYPE_FIELD;
2349                     s->mb_intra= 0;
2350                     for(dir=0; dir<2; dir++){
2351                         for(i=0; i<2; i++){
2352                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2353                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2354                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2355                         }
2356                     }
2357                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2358                                  &dmin, &next_block, 0, 0);
2359                 }
2360                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2361                     s->mv_dir = 0;
2362                     s->mv_type = MV_TYPE_16X16;
2363                     s->mb_intra= 1;
2364                     s->mv[0][0][0] = 0;
2365                     s->mv[0][0][1] = 0;
2366                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2367                                  &dmin, &next_block, 0, 0);
2368                     if(s->h263_pred || s->h263_aic){
2369                         if(best_s.mb_intra)
2370                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2371                         else
2372                             ff_clean_intra_table_entries(s); //old mode?
2373                     }
2374                 }
2375
2376                 if((s->flags & CODEC_FLAG_QP_RD) && dmin < INT_MAX){
2377                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2378                         const int last_qp= backup_s.qscale;
2379                         int qpi, qp, dc[6];
2380                         DCTELEM ac[6][16];
2381                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2382                         static const int dquant_tab[4]={-1,1,-2,2};
2383
2384                         assert(backup_s.dquant == 0);
2385
2386                         //FIXME intra
2387                         s->mv_dir= best_s.mv_dir;
2388                         s->mv_type = MV_TYPE_16X16;
2389                         s->mb_intra= best_s.mb_intra;
2390                         s->mv[0][0][0] = best_s.mv[0][0][0];
2391                         s->mv[0][0][1] = best_s.mv[0][0][1];
2392                         s->mv[1][0][0] = best_s.mv[1][0][0];
2393                         s->mv[1][0][1] = best_s.mv[1][0][1];
2394
2395                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
2396                         for(; qpi<4; qpi++){
2397                             int dquant= dquant_tab[qpi];
2398                             qp= last_qp + dquant;
2399                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2400                                 continue;
2401                             backup_s.dquant= dquant;
2402                             if(s->mb_intra && s->dc_val[0]){
2403                                 for(i=0; i<6; i++){
2404                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2405                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(DCTELEM)*16);
2406                                 }
2407                             }
2408
2409                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2410                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2411                             if(best_s.qscale != qp){
2412                                 if(s->mb_intra && s->dc_val[0]){
2413                                     for(i=0; i<6; i++){
2414                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2415                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(DCTELEM)*16);
2416                                     }
2417                                 }
2418                             }
2419                         }
2420                     }
2421                 }
2422                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2423                     int mx= s->b_direct_mv_table[xy][0];
2424                     int my= s->b_direct_mv_table[xy][1];
2425
2426                     backup_s.dquant = 0;
2427                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2428                     s->mb_intra= 0;
2429                     ff_mpeg4_set_direct_mv(s, mx, my);
2430                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2431                                  &dmin, &next_block, mx, my);
2432                 }
2433                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2434                     backup_s.dquant = 0;
2435                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2436                     s->mb_intra= 0;
2437                     ff_mpeg4_set_direct_mv(s, 0, 0);
2438                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2439                                  &dmin, &next_block, 0, 0);
2440                 }
2441                 if(!best_s.mb_intra && s->flags2&CODEC_FLAG2_SKIP_RD){
2442                     int coded=0;
2443                     for(i=0; i<6; i++)
2444                         coded |= s->block_last_index[i];
2445                     if(coded){
2446                         int mx,my;
2447                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2448                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
2449                             mx=my=0; //FIXME find the one we actually used
2450                             ff_mpeg4_set_direct_mv(s, mx, my);
2451                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2452                             mx= s->mv[1][0][0];
2453                             my= s->mv[1][0][1];
2454                         }else{
2455                             mx= s->mv[0][0][0];
2456                             my= s->mv[0][0][1];
2457                         }
2458
2459                         s->mv_dir= best_s.mv_dir;
2460                         s->mv_type = best_s.mv_type;
2461                         s->mb_intra= 0;
2462 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2463                         s->mv[0][0][1] = best_s.mv[0][0][1];
2464                         s->mv[1][0][0] = best_s.mv[1][0][0];
2465                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2466                         backup_s.dquant= 0;
2467                         s->skipdct=1;
2468                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2469                                         &dmin, &next_block, mx, my);
2470                         s->skipdct=0;
2471                     }
2472                 }
2473
2474                 s->current_picture.qscale_table[xy]= best_s.qscale;
2475
2476                 copy_context_after_encode(s, &best_s, -1);
2477
2478                 pb_bits_count= put_bits_count(&s->pb);
2479                 flush_put_bits(&s->pb);
2480                 ff_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2481                 s->pb= backup_s.pb;
2482
2483                 if(s->data_partitioning){
2484                     pb2_bits_count= put_bits_count(&s->pb2);
2485                     flush_put_bits(&s->pb2);
2486                     ff_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2487                     s->pb2= backup_s.pb2;
2488
2489                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2490                     flush_put_bits(&s->tex_pb);
2491                     ff_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2492                     s->tex_pb= backup_s.tex_pb;
2493                 }
2494                 s->last_bits= put_bits_count(&s->pb);
2495
2496                 if (CONFIG_H263_ENCODER &&
2497                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2498                     ff_h263_update_motion_val(s);
2499
2500                 if(next_block==0){ //FIXME 16 vs linesize16
2501                     s->dsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
2502                     s->dsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2503                     s->dsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2504                 }
2505
2506                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
2507                     MPV_decode_mb(s, s->block);
2508             } else {
2509                 int motion_x = 0, motion_y = 0;
2510                 s->mv_type=MV_TYPE_16X16;
2511                 // only one MB-Type possible
2512
2513                 switch(mb_type){
2514                 case CANDIDATE_MB_TYPE_INTRA:
2515                     s->mv_dir = 0;
2516                     s->mb_intra= 1;
2517                     motion_x= s->mv[0][0][0] = 0;
2518                     motion_y= s->mv[0][0][1] = 0;
2519                     break;
2520                 case CANDIDATE_MB_TYPE_INTER:
2521                     s->mv_dir = MV_DIR_FORWARD;
2522                     s->mb_intra= 0;
2523                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
2524                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
2525                     break;
2526                 case CANDIDATE_MB_TYPE_INTER_I:
2527                     s->mv_dir = MV_DIR_FORWARD;
2528                     s->mv_type = MV_TYPE_FIELD;
2529                     s->mb_intra= 0;
2530                     for(i=0; i<2; i++){
2531                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2532                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2533                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2534                     }
2535                     break;
2536                 case CANDIDATE_MB_TYPE_INTER4V:
2537                     s->mv_dir = MV_DIR_FORWARD;
2538                     s->mv_type = MV_TYPE_8X8;
2539                     s->mb_intra= 0;
2540                     for(i=0; i<4; i++){
2541                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2542                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2543                     }
2544                     break;
2545                 case CANDIDATE_MB_TYPE_DIRECT:
2546                     if (CONFIG_MPEG4_ENCODER) {
2547                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2548                         s->mb_intra= 0;
2549                         motion_x=s->b_direct_mv_table[xy][0];
2550                         motion_y=s->b_direct_mv_table[xy][1];
2551                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
2552                     }
2553                     break;
2554                 case CANDIDATE_MB_TYPE_DIRECT0:
2555                     if (CONFIG_MPEG4_ENCODER) {
2556                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2557                         s->mb_intra= 0;
2558                         ff_mpeg4_set_direct_mv(s, 0, 0);
2559                     }
2560                     break;
2561                 case CANDIDATE_MB_TYPE_BIDIR:
2562                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2563                     s->mb_intra= 0;
2564                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2565                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2566                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2567                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2568                     break;
2569                 case CANDIDATE_MB_TYPE_BACKWARD:
2570                     s->mv_dir = MV_DIR_BACKWARD;
2571                     s->mb_intra= 0;
2572                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2573                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2574                     break;
2575                 case CANDIDATE_MB_TYPE_FORWARD:
2576                     s->mv_dir = MV_DIR_FORWARD;
2577                     s->mb_intra= 0;
2578                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2579                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2580 //                    printf(" %d %d ", motion_x, motion_y);
2581                     break;
2582                 case CANDIDATE_MB_TYPE_FORWARD_I:
2583                     s->mv_dir = MV_DIR_FORWARD;
2584                     s->mv_type = MV_TYPE_FIELD;
2585                     s->mb_intra= 0;
2586                     for(i=0; i<2; i++){
2587                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2588                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2589                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2590                     }
2591                     break;
2592                 case CANDIDATE_MB_TYPE_BACKWARD_I:
2593                     s->mv_dir = MV_DIR_BACKWARD;
2594                     s->mv_type = MV_TYPE_FIELD;
2595                     s->mb_intra= 0;
2596                     for(i=0; i<2; i++){
2597                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2598                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2599                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2600                     }
2601                     break;
2602                 case CANDIDATE_MB_TYPE_BIDIR_I:
2603                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2604                     s->mv_type = MV_TYPE_FIELD;
2605                     s->mb_intra= 0;
2606                     for(dir=0; dir<2; dir++){
2607                         for(i=0; i<2; i++){
2608                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2609                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2610                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2611                         }
2612                     }
2613                     break;
2614                 default:
2615                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
2616                 }
2617
2618                 encode_mb(s, motion_x, motion_y);
2619
2620                 // RAL: Update last macroblock type
2621                 s->last_mv_dir = s->mv_dir;
2622
2623                 if (CONFIG_H263_ENCODER &&
2624                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2625                     ff_h263_update_motion_val(s);
2626
2627                 MPV_decode_mb(s, s->block);
2628             }
2629
2630             /* clean the MV table in IPS frames for direct mode in B frames */
2631             if(s->mb_intra /* && I,P,S_TYPE */){
2632                 s->p_mv_table[xy][0]=0;
2633                 s->p_mv_table[xy][1]=0;
2634             }
2635
2636             if(s->flags&CODEC_FLAG_PSNR){
2637                 int w= 16;
2638                 int h= 16;
2639
2640                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2641                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2642
2643                 s->current_picture.error[0] += sse(
2644                     s, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
2645                     s->dest[0], w, h, s->linesize);
2646                 s->current_picture.error[1] += sse(
2647                     s, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2648                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2649                 s->current_picture.error[2] += sse(
2650                     s, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2651                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2652             }
2653             if(s->loop_filter){
2654                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
2655                     ff_h263_loop_filter(s);
2656             }
2657 //printf("MB %d %d bits\n", s->mb_x+s->mb_y*s->mb_stride, put_bits_count(&s->pb));
2658         }
2659     }
2660
2661     //not beautiful here but we must write it before flushing so it has to be here
2662     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
2663         msmpeg4_encode_ext_header(s);
2664
2665     write_slice_end(s);
2666
2667     /* Send the last GOB if RTP */
2668     if (s->avctx->rtp_callback) {
2669         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
2670         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
2671         /* Call the RTP callback to send the last GOB */
2672         emms_c();
2673         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
2674     }
2675
2676     return 0;
2677 }
2678
2679 #define MERGE(field) dst->field += src->field; src->field=0
2680 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
2681     MERGE(me.scene_change_score);
2682     MERGE(me.mc_mb_var_sum_temp);
2683     MERGE(me.mb_var_sum_temp);
2684 }
2685
2686 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
2687     int i;
2688
2689     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
2690     MERGE(dct_count[1]);
2691     MERGE(mv_bits);
2692     MERGE(i_tex_bits);
2693     MERGE(p_tex_bits);
2694     MERGE(i_count);
2695     MERGE(f_count);
2696     MERGE(b_count);
2697     MERGE(skip_count);
2698     MERGE(misc_bits);
2699     MERGE(error_count);
2700     MERGE(padding_bug_score);
2701     MERGE(current_picture.error[0]);
2702     MERGE(current_picture.error[1]);
2703     MERGE(current_picture.error[2]);
2704
2705     if(dst->avctx->noise_reduction){
2706         for(i=0; i<64; i++){
2707             MERGE(dct_error_sum[0][i]);
2708             MERGE(dct_error_sum[1][i]);
2709         }
2710     }
2711
2712     assert(put_bits_count(&src->pb) % 8 ==0);
2713     assert(put_bits_count(&dst->pb) % 8 ==0);
2714     ff_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
2715     flush_put_bits(&dst->pb);
2716 }
2717
2718 static int estimate_qp(MpegEncContext *s, int dry_run){
2719     if (s->next_lambda){
2720         s->current_picture_ptr->quality=
2721         s->current_picture.quality = s->next_lambda;
2722         if(!dry_run) s->next_lambda= 0;
2723     } else if (!s->fixed_qscale) {
2724         s->current_picture_ptr->quality=
2725         s->current_picture.quality = ff_rate_estimate_qscale(s, dry_run);
2726         if (s->current_picture.quality < 0)
2727             return -1;
2728     }
2729
2730     if(s->adaptive_quant){
2731         switch(s->codec_id){
2732         case CODEC_ID_MPEG4:
2733             if (CONFIG_MPEG4_ENCODER)
2734                 ff_clean_mpeg4_qscales(s);
2735             break;
2736         case CODEC_ID_H263:
2737         case CODEC_ID_H263P:
2738         case CODEC_ID_FLV1:
2739             if (CONFIG_H263_ENCODER)
2740                 ff_clean_h263_qscales(s);
2741             break;
2742         default:
2743             ff_init_qscale_tab(s);
2744         }
2745
2746         s->lambda= s->lambda_table[0];
2747         //FIXME broken
2748     }else
2749         s->lambda= s->current_picture.quality;
2750 //printf("%d %d\n", s->avctx->global_quality, s->current_picture.quality);
2751     update_qscale(s);
2752     return 0;
2753 }
2754
2755 /* must be called before writing the header */
2756 static void set_frame_distances(MpegEncContext * s){
2757     assert(s->current_picture_ptr->pts != AV_NOPTS_VALUE);
2758     s->time= s->current_picture_ptr->pts*s->avctx->time_base.num;
2759
2760     if(s->pict_type==AV_PICTURE_TYPE_B){
2761         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
2762         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
2763     }else{
2764         s->pp_time= s->time - s->last_non_b_time;
2765         s->last_non_b_time= s->time;
2766         assert(s->picture_number==0 || s->pp_time > 0);
2767     }
2768 }
2769
2770 static int encode_picture(MpegEncContext *s, int picture_number)
2771 {
2772     int i;
2773     int bits;
2774
2775     s->picture_number = picture_number;
2776
2777     /* Reset the average MB variance */
2778     s->me.mb_var_sum_temp    =
2779     s->me.mc_mb_var_sum_temp = 0;
2780
2781     /* we need to initialize some time vars before we can encode b-frames */
2782     // RAL: Condition added for MPEG1VIDEO
2783     if (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->h263_msmpeg4))
2784         set_frame_distances(s);
2785     if(CONFIG_MPEG4_ENCODER && s->codec_id == CODEC_ID_MPEG4)
2786         ff_set_mpeg4_time(s);
2787
2788     s->me.scene_change_score=0;
2789
2790 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
2791
2792     if(s->pict_type==AV_PICTURE_TYPE_I){
2793         if(s->msmpeg4_version >= 3) s->no_rounding=1;
2794         else                        s->no_rounding=0;
2795     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
2796         if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
2797             s->no_rounding ^= 1;
2798     }
2799
2800     if(s->flags & CODEC_FLAG_PASS2){
2801         if (estimate_qp(s,1) < 0)
2802             return -1;
2803         ff_get_2pass_fcode(s);
2804     }else if(!(s->flags & CODEC_FLAG_QSCALE)){
2805         if(s->pict_type==AV_PICTURE_TYPE_B)
2806             s->lambda= s->last_lambda_for[s->pict_type];
2807         else
2808             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
2809         update_qscale(s);
2810     }
2811
2812     s->mb_intra=0; //for the rate distortion & bit compare functions
2813     for(i=1; i<s->avctx->thread_count; i++){
2814         ff_update_duplicate_context(s->thread_context[i], s);
2815     }
2816
2817     if(ff_init_me(s)<0)
2818         return -1;
2819
2820     /* Estimate motion for every MB */
2821     if(s->pict_type != AV_PICTURE_TYPE_I){
2822         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
2823         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
2824         if(s->pict_type != AV_PICTURE_TYPE_B && s->avctx->me_threshold==0){
2825             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
2826                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, s->avctx->thread_count, sizeof(void*));
2827             }
2828         }
2829
2830         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, s->avctx->thread_count, sizeof(void*));
2831     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
2832         /* I-Frame */
2833         for(i=0; i<s->mb_stride*s->mb_height; i++)
2834             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
2835
2836         if(!s->fixed_qscale){
2837             /* finding spatial complexity for I-frame rate control */
2838             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, s->avctx->thread_count, sizeof(void*));
2839         }
2840     }
2841     for(i=1; i<s->avctx->thread_count; i++){
2842         merge_context_after_me(s, s->thread_context[i]);
2843     }
2844     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
2845     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
2846     emms_c();
2847
2848     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
2849         s->pict_type= AV_PICTURE_TYPE_I;
2850         for(i=0; i<s->mb_stride*s->mb_height; i++)
2851             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
2852 //printf("Scene change detected, encoding as I Frame %d %d\n", s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
2853     }
2854
2855     if(!s->umvplus){
2856         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
2857             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
2858
2859             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2860                 int a,b;
2861                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
2862                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
2863                 s->f_code= FFMAX3(s->f_code, a, b);
2864             }
2865
2866             ff_fix_long_p_mvs(s);
2867             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
2868             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2869                 int j;
2870                 for(i=0; i<2; i++){
2871                     for(j=0; j<2; j++)
2872                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
2873                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
2874                 }
2875             }
2876         }
2877
2878         if(s->pict_type==AV_PICTURE_TYPE_B){
2879             int a, b;
2880
2881             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
2882             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
2883             s->f_code = FFMAX(a, b);
2884
2885             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
2886             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
2887             s->b_code = FFMAX(a, b);
2888
2889             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
2890             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
2891             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
2892             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
2893             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2894                 int dir, j;
2895                 for(dir=0; dir<2; dir++){
2896                     for(i=0; i<2; i++){
2897                         for(j=0; j<2; j++){
2898                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
2899                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
2900                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
2901                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
2902                         }
2903                     }
2904                 }
2905             }
2906         }
2907     }
2908
2909     if (estimate_qp(s, 0) < 0)
2910         return -1;
2911
2912     if(s->qscale < 3 && s->max_qcoeff<=128 && s->pict_type==AV_PICTURE_TYPE_I && !(s->flags & CODEC_FLAG_QSCALE))
2913         s->qscale= 3; //reduce clipping problems
2914
2915     if (s->out_format == FMT_MJPEG) {
2916         /* for mjpeg, we do include qscale in the matrix */
2917         for(i=1;i<64;i++){
2918             int j= s->dsp.idct_permutation[i];
2919
2920             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
2921         }
2922         s->y_dc_scale_table=
2923         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
2924         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
2925         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
2926                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
2927         s->qscale= 8;
2928     }
2929
2930     //FIXME var duplication
2931     s->current_picture_ptr->key_frame=
2932     s->current_picture.key_frame= s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
2933     s->current_picture_ptr->pict_type=
2934     s->current_picture.pict_type= s->pict_type;
2935
2936     if(s->current_picture.key_frame)
2937         s->picture_in_gop_number=0;
2938
2939     s->last_bits= put_bits_count(&s->pb);
2940     switch(s->out_format) {
2941     case FMT_MJPEG:
2942         if (CONFIG_MJPEG_ENCODER)
2943             ff_mjpeg_encode_picture_header(s);
2944         break;
2945     case FMT_H261:
2946         if (CONFIG_H261_ENCODER)
2947             ff_h261_encode_picture_header(s, picture_number);
2948         break;
2949     case FMT_H263:
2950         if (CONFIG_WMV2_ENCODER && s->codec_id == CODEC_ID_WMV2)
2951             ff_wmv2_encode_picture_header(s, picture_number);
2952         else if (CONFIG_MSMPEG4_ENCODER && s->h263_msmpeg4)
2953             msmpeg4_encode_picture_header(s, picture_number);
2954         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
2955             mpeg4_encode_picture_header(s, picture_number);
2956         else if (CONFIG_RV10_ENCODER && s->codec_id == CODEC_ID_RV10)
2957             rv10_encode_picture_header(s, picture_number);
2958         else if (CONFIG_RV20_ENCODER && s->codec_id == CODEC_ID_RV20)
2959             rv20_encode_picture_header(s, picture_number);
2960         else if (CONFIG_FLV_ENCODER && s->codec_id == CODEC_ID_FLV1)
2961             ff_flv_encode_picture_header(s, picture_number);
2962         else if (CONFIG_H263_ENCODER)
2963             h263_encode_picture_header(s, picture_number);
2964         break;
2965     case FMT_MPEG1:
2966         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2967             mpeg1_encode_picture_header(s, picture_number);
2968         break;
2969     case FMT_H264:
2970         break;
2971     default:
2972         assert(0);
2973     }
2974     bits= put_bits_count(&s->pb);
2975     s->header_bits= bits - s->last_bits;
2976
2977     for(i=1; i<s->avctx->thread_count; i++){
2978         update_duplicate_context_after_me(s->thread_context[i], s);
2979     }
2980     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, s->avctx->thread_count, sizeof(void*));
2981     for(i=1; i<s->avctx->thread_count; i++){
2982         merge_context_after_encode(s, s->thread_context[i]);
2983     }
2984     emms_c();
2985     return 0;
2986 }
2987
2988 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block){
2989     const int intra= s->mb_intra;
2990     int i;
2991
2992     s->dct_count[intra]++;
2993
2994     for(i=0; i<64; i++){
2995         int level= block[i];
2996
2997         if(level){
2998             if(level>0){
2999                 s->dct_error_sum[intra][i] += level;
3000                 level -= s->dct_offset[intra][i];
3001                 if(level<0) level=0;
3002             }else{
3003                 s->dct_error_sum[intra][i] -= level;
3004                 level += s->dct_offset[intra][i];
3005                 if(level>0) level=0;
3006             }
3007             block[i]= level;
3008         }
3009     }
3010 }
3011
3012 static int dct_quantize_trellis_c(MpegEncContext *s,
3013                                   DCTELEM *block, int n,
3014                                   int qscale, int *overflow){
3015     const int *qmat;
3016     const uint8_t *scantable= s->intra_scantable.scantable;
3017     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3018     int max=0;
3019     unsigned int threshold1, threshold2;
3020     int bias=0;
3021     int run_tab[65];
3022     int level_tab[65];
3023     int score_tab[65];
3024     int survivor[65];
3025     int survivor_count;
3026     int last_run=0;
3027     int last_level=0;
3028     int last_score= 0;
3029     int last_i;
3030     int coeff[2][64];
3031     int coeff_count[64];
3032     int qmul, qadd, start_i, last_non_zero, i, dc;
3033     const int esc_length= s->ac_esc_length;
3034     uint8_t * length;
3035     uint8_t * last_length;
3036     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3037
3038     s->dsp.fdct (block);
3039
3040     if(s->dct_error_sum)
3041         s->denoise_dct(s, block);
3042     qmul= qscale*16;
3043     qadd= ((qscale-1)|1)*8;
3044
3045     if (s->mb_intra) {
3046         int q;
3047         if (!s->h263_aic) {
3048             if (n < 4)
3049                 q = s->y_dc_scale;
3050             else
3051                 q = s->c_dc_scale;
3052             q = q << 3;
3053         } else{
3054             /* For AIC we skip quant/dequant of INTRADC */
3055             q = 1 << 3;
3056             qadd=0;
3057         }
3058
3059         /* note: block[0] is assumed to be positive */
3060         block[0] = (block[0] + (q >> 1)) / q;
3061         start_i = 1;
3062         last_non_zero = 0;
3063         qmat = s->q_intra_matrix[qscale];
3064         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3065             bias= 1<<(QMAT_SHIFT-1);
3066         length     = s->intra_ac_vlc_length;
3067         last_length= s->intra_ac_vlc_last_length;
3068     } else {
3069         start_i = 0;
3070         last_non_zero = -1;
3071         qmat = s->q_inter_matrix[qscale];
3072         length     = s->inter_ac_vlc_length;
3073         last_length= s->inter_ac_vlc_last_length;
3074     }
3075     last_i= start_i;
3076
3077     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3078     threshold2= (threshold1<<1);
3079
3080     for(i=63; i>=start_i; i--) {
3081         const int j = scantable[i];
3082         int level = block[j] * qmat[j];
3083
3084         if(((unsigned)(level+threshold1))>threshold2){
3085             last_non_zero = i;
3086             break;
3087         }
3088     }
3089
3090     for(i=start_i; i<=last_non_zero; i++) {
3091         const int j = scantable[i];
3092         int level = block[j] * qmat[j];
3093
3094 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3095 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3096         if(((unsigned)(level+threshold1))>threshold2){
3097             if(level>0){
3098                 level= (bias + level)>>QMAT_SHIFT;
3099                 coeff[0][i]= level;
3100                 coeff[1][i]= level-1;
3101 //                coeff[2][k]= level-2;
3102             }else{
3103                 level= (bias - level)>>QMAT_SHIFT;
3104                 coeff[0][i]= -level;
3105                 coeff[1][i]= -level+1;
3106 //                coeff[2][k]= -level+2;
3107             }
3108             coeff_count[i]= FFMIN(level, 2);
3109             assert(coeff_count[i]);
3110             max |=level;
3111         }else{
3112             coeff[0][i]= (level>>31)|1;
3113             coeff_count[i]= 1;
3114         }
3115     }
3116
3117     *overflow= s->max_qcoeff < max; //overflow might have happened
3118
3119     if(last_non_zero < start_i){
3120         memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3121         return last_non_zero;
3122     }
3123
3124     score_tab[start_i]= 0;
3125     survivor[0]= start_i;
3126     survivor_count= 1;
3127
3128     for(i=start_i; i<=last_non_zero; i++){
3129         int level_index, j, zero_distortion;
3130         int dct_coeff= FFABS(block[ scantable[i] ]);
3131         int best_score=256*256*256*120;
3132
3133         if (   s->dsp.fdct == fdct_ifast
3134 #ifndef FAAN_POSTSCALE
3135             || s->dsp.fdct == ff_faandct
3136 #endif
3137            )
3138             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3139         zero_distortion= dct_coeff*dct_coeff;
3140
3141         for(level_index=0; level_index < coeff_count[i]; level_index++){
3142             int distortion;
3143             int level= coeff[level_index][i];
3144             const int alevel= FFABS(level);
3145             int unquant_coeff;
3146
3147             assert(level);
3148
3149             if(s->out_format == FMT_H263){
3150                 unquant_coeff= alevel*qmul + qadd;
3151             }else{ //MPEG1
3152                 j= s->dsp.idct_permutation[ scantable[i] ]; //FIXME optimize
3153                 if(s->mb_intra){
3154                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3155                         unquant_coeff =   (unquant_coeff - 1) | 1;
3156                 }else{
3157                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3158                         unquant_coeff =   (unquant_coeff - 1) | 1;
3159                 }
3160                 unquant_coeff<<= 3;
3161             }
3162
3163             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3164             level+=64;
3165             if((level&(~127)) == 0){
3166                 for(j=survivor_count-1; j>=0; j--){
3167                     int run= i - survivor[j];
3168                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3169                     score += score_tab[i-run];
3170
3171                     if(score < best_score){
3172                         best_score= score;
3173                         run_tab[i+1]= run;
3174                         level_tab[i+1]= level-64;
3175                     }
3176                 }
3177
3178                 if(s->out_format == FMT_H263){
3179                     for(j=survivor_count-1; j>=0; j--){
3180                         int run= i - survivor[j];
3181                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3182                         score += score_tab[i-run];
3183                         if(score < last_score){
3184                             last_score= score;
3185                             last_run= run;
3186                             last_level= level-64;
3187                             last_i= i+1;
3188                         }
3189                     }
3190                 }
3191             }else{
3192                 distortion += esc_length*lambda;
3193                 for(j=survivor_count-1; j>=0; j--){
3194                     int run= i - survivor[j];
3195                     int score= distortion + score_tab[i-run];
3196
3197                     if(score < best_score){
3198                         best_score= score;
3199                         run_tab[i+1]= run;
3200                         level_tab[i+1]= level-64;
3201                     }
3202                 }
3203
3204                 if(s->out_format == FMT_H263){
3205                   for(j=survivor_count-1; j>=0; j--){
3206                         int run= i - survivor[j];
3207                         int score= distortion + score_tab[i-run];
3208                         if(score < last_score){
3209                             last_score= score;
3210                             last_run= run;
3211                             last_level= level-64;
3212                             last_i= i+1;
3213                         }
3214                     }
3215                 }
3216             }
3217         }
3218
3219         score_tab[i+1]= best_score;
3220
3221         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
3222         if(last_non_zero <= 27){
3223             for(; survivor_count; survivor_count--){
3224                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3225                     break;
3226             }
3227         }else{
3228             for(; survivor_count; survivor_count--){
3229                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3230                     break;
3231             }
3232         }
3233
3234         survivor[ survivor_count++ ]= i+1;
3235     }
3236
3237     if(s->out_format != FMT_H263){
3238         last_score= 256*256*256*120;
3239         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3240             int score= score_tab[i];
3241             if(i) score += lambda*2; //FIXME exacter?
3242
3243             if(score < last_score){
3244                 last_score= score;
3245                 last_i= i;
3246                 last_level= level_tab[i];
3247                 last_run= run_tab[i];
3248             }
3249         }
3250     }
3251
3252     s->coded_score[n] = last_score;
3253
3254     dc= FFABS(block[0]);
3255     last_non_zero= last_i - 1;
3256     memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3257
3258     if(last_non_zero < start_i)
3259         return last_non_zero;
3260
3261     if(last_non_zero == 0 && start_i == 0){
3262         int best_level= 0;
3263         int best_score= dc * dc;
3264
3265         for(i=0; i<coeff_count[0]; i++){
3266             int level= coeff[i][0];
3267             int alevel= FFABS(level);
3268             int unquant_coeff, score, distortion;
3269
3270             if(s->out_format == FMT_H263){
3271                     unquant_coeff= (alevel*qmul + qadd)>>3;
3272             }else{ //MPEG1
3273                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3274                     unquant_coeff =   (unquant_coeff - 1) | 1;
3275             }
3276             unquant_coeff = (unquant_coeff + 4) >> 3;
3277             unquant_coeff<<= 3 + 3;
3278
3279             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3280             level+=64;
3281             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3282             else                    score= distortion + esc_length*lambda;
3283
3284             if(score < best_score){
3285                 best_score= score;
3286                 best_level= level - 64;
3287             }
3288         }
3289         block[0]= best_level;
3290         s->coded_score[n] = best_score - dc*dc;
3291         if(best_level == 0) return -1;
3292         else                return last_non_zero;
3293     }
3294
3295     i= last_i;
3296     assert(last_level);
3297
3298     block[ perm_scantable[last_non_zero] ]= last_level;
3299     i -= last_run + 1;
3300
3301     for(; i>start_i; i -= run_tab[i] + 1){
3302         block[ perm_scantable[i-1] ]= level_tab[i];
3303     }
3304
3305     return last_non_zero;
3306 }
3307
3308 //#define REFINE_STATS 1
3309 static int16_t basis[64][64];
3310
3311 static void build_basis(uint8_t *perm){
3312     int i, j, x, y;
3313     emms_c();
3314     for(i=0; i<8; i++){
3315         for(j=0; j<8; j++){
3316             for(y=0; y<8; y++){
3317                 for(x=0; x<8; x++){
3318                     double s= 0.25*(1<<BASIS_SHIFT);
3319                     int index= 8*i + j;
3320                     int perm_index= perm[index];
3321                     if(i==0) s*= sqrt(0.5);
3322                     if(j==0) s*= sqrt(0.5);
3323                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3324                 }
3325             }
3326         }
3327     }
3328 }
3329
3330 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3331                         DCTELEM *block, int16_t *weight, DCTELEM *orig,
3332                         int n, int qscale){
3333     int16_t rem[64];
3334     LOCAL_ALIGNED_16(DCTELEM, d1, [64]);
3335     const uint8_t *scantable= s->intra_scantable.scantable;
3336     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3337 //    unsigned int threshold1, threshold2;
3338 //    int bias=0;
3339     int run_tab[65];
3340     int prev_run=0;
3341     int prev_level=0;
3342     int qmul, qadd, start_i, last_non_zero, i, dc;
3343     uint8_t * length;
3344     uint8_t * last_length;
3345     int lambda;
3346     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3347 #ifdef REFINE_STATS
3348 static int count=0;
3349 static int after_last=0;
3350 static int to_zero=0;
3351 static int from_zero=0;
3352 static int raise=0;
3353 static int lower=0;
3354 static int messed_sign=0;
3355 #endif
3356
3357     if(basis[0][0] == 0)
3358         build_basis(s->dsp.idct_permutation);
3359
3360     qmul= qscale*2;
3361     qadd= (qscale-1)|1;
3362     if (s->mb_intra) {
3363         if (!s->h263_aic) {
3364             if (n < 4)
3365                 q = s->y_dc_scale;
3366             else
3367                 q = s->c_dc_scale;
3368         } else{
3369             /* For AIC we skip quant/dequant of INTRADC */
3370             q = 1;