move mpeg encoder specific initialization in the encoder specific file
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard.
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * This file is part of FFmpeg.
7  *
8  * FFmpeg is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * FFmpeg is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with FFmpeg; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  *
22  * 4MV & hq & b-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
23  */
24
25 /**
26  * @file mpegvideo_enc.c
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include "avcodec.h"
31 #include "dsputil.h"
32 #include "mpegvideo.h"
33 #include "mpegvideo_common.h"
34 #include "mjpegenc.h"
35 #include "msmpeg4.h"
36 #include "faandct.h"
37 #include <limits.h>
38
39 //#undef NDEBUG
40 //#include <assert.h>
41
42 static int encode_picture(MpegEncContext *s, int picture_number);
43 static int dct_quantize_refine(MpegEncContext *s, DCTELEM *block, int16_t *weight, DCTELEM *orig, int n, int qscale);
44 static int sse_mb(MpegEncContext *s);
45
46 /* enable all paranoid tests for rounding, overflows, etc... */
47 //#define PARANOID
48
49 //#define DEBUG
50
51 static const uint16_t aanscales[64] = {
52     /* precomputed values scaled up by 14 bits */
53     16384, 22725, 21407, 19266, 16384, 12873,  8867,  4520,
54     22725, 31521, 29692, 26722, 22725, 17855, 12299,  6270,
55     21407, 29692, 27969, 25172, 21407, 16819, 11585,  5906,
56     19266, 26722, 25172, 22654, 19266, 15137, 10426,  5315,
57     16384, 22725, 21407, 19266, 16384, 12873,  8867,  4520,
58     12873, 17855, 16819, 15137, 12873, 10114,  6967,  3552,
59     8867 , 12299, 11585, 10426,  8867,  6967,  4799,  2446,
60     4520 ,  6270,  5906,  5315,  4520,  3552,  2446,  1247
61 };
62
63 static uint8_t default_mv_penalty[MAX_FCODE+1][MAX_MV*2+1];
64 static uint8_t default_fcode_tab[MAX_MV*2+1];
65
66 enum PixelFormat ff_yuv420p_list[2]= {PIX_FMT_YUV420P, -1};
67
68 static void convert_matrix(DSPContext *dsp, int (*qmat)[64], uint16_t (*qmat16)[2][64],
69                            const uint16_t *quant_matrix, int bias, int qmin, int qmax, int intra)
70 {
71     int qscale;
72     int shift=0;
73
74     for(qscale=qmin; qscale<=qmax; qscale++){
75         int i;
76         if (dsp->fdct == ff_jpeg_fdct_islow
77 #ifdef FAAN_POSTSCALE
78             || dsp->fdct == ff_faandct
79 #endif
80             ) {
81             for(i=0;i<64;i++) {
82                 const int j= dsp->idct_permutation[i];
83                 /* 16 <= qscale * quant_matrix[i] <= 7905 */
84                 /* 19952         <= aanscales[i] * qscale * quant_matrix[i]           <= 249205026 */
85                 /* (1<<36)/19952 >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= (1<<36)/249205026 */
86                 /* 3444240       >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= 275 */
87
88                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
89                                 (qscale * quant_matrix[j]));
90             }
91         } else if (dsp->fdct == fdct_ifast
92 #ifndef FAAN_POSTSCALE
93                    || dsp->fdct == ff_faandct
94 #endif
95                    ) {
96             for(i=0;i<64;i++) {
97                 const int j= dsp->idct_permutation[i];
98                 /* 16 <= qscale * quant_matrix[i] <= 7905 */
99                 /* 19952         <= aanscales[i] * qscale * quant_matrix[i]           <= 249205026 */
100                 /* (1<<36)/19952 >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= (1<<36)/249205026 */
101                 /* 3444240       >= (1<<36)/(aanscales[i] * qscale * quant_matrix[i]) >= 275 */
102
103                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) /
104                                 (aanscales[i] * qscale * quant_matrix[j]));
105             }
106         } else {
107             for(i=0;i<64;i++) {
108                 const int j= dsp->idct_permutation[i];
109                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
110                    So 16           <= qscale * quant_matrix[i]             <= 7905
111                    so (1<<19) / 16 >= (1<<19) / (qscale * quant_matrix[i]) >= (1<<19) / 7905
112                    so 32768        >= (1<<19) / (qscale * quant_matrix[i]) >= 67
113                 */
114                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / (qscale * quant_matrix[j]));
115 //                qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) / (qscale * quant_matrix[i]);
116                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / (qscale * quant_matrix[j]);
117
118                 if(qmat16[qscale][0][i]==0 || qmat16[qscale][0][i]==128*256) qmat16[qscale][0][i]=128*256-1;
119                 qmat16[qscale][1][i]= ROUNDED_DIV(bias<<(16-QUANT_BIAS_SHIFT), qmat16[qscale][0][i]);
120             }
121         }
122
123         for(i=intra; i<64; i++){
124             int64_t max= 8191;
125             if (dsp->fdct == fdct_ifast
126 #ifndef FAAN_POSTSCALE
127                    || dsp->fdct == ff_faandct
128 #endif
129                    ) {
130                 max= (8191LL*aanscales[i]) >> 14;
131             }
132             while(((max * qmat[qscale][i]) >> shift) > INT_MAX){
133                 shift++;
134             }
135         }
136     }
137     if(shift){
138         av_log(NULL, AV_LOG_INFO, "Warning, QMAT_SHIFT is larger than %d, overflows possible\n", QMAT_SHIFT - shift);
139     }
140 }
141
142 static inline void update_qscale(MpegEncContext *s){
143     s->qscale= (s->lambda*139 + FF_LAMBDA_SCALE*64) >> (FF_LAMBDA_SHIFT + 7);
144     s->qscale= av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
145
146     s->lambda2= (s->lambda*s->lambda + FF_LAMBDA_SCALE/2) >> FF_LAMBDA_SHIFT;
147 }
148
149 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix){
150     int i;
151
152     if(matrix){
153         put_bits(pb, 1, 1);
154         for(i=0;i<64;i++) {
155             put_bits(pb, 8, matrix[ ff_zigzag_direct[i] ]);
156         }
157     }else
158         put_bits(pb, 1, 0);
159 }
160
161 static void copy_picture_attributes(MpegEncContext *s, AVFrame *dst, AVFrame *src){
162     int i;
163
164     dst->pict_type              = src->pict_type;
165     dst->quality                = src->quality;
166     dst->coded_picture_number   = src->coded_picture_number;
167     dst->display_picture_number = src->display_picture_number;
168 //    dst->reference              = src->reference;
169     dst->pts                    = src->pts;
170     dst->interlaced_frame       = src->interlaced_frame;
171     dst->top_field_first        = src->top_field_first;
172
173     if(s->avctx->me_threshold){
174         if(!src->motion_val[0])
175             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_val not set!\n");
176         if(!src->mb_type)
177             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.mb_type not set!\n");
178         if(!src->ref_index[0])
179             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.ref_index not set!\n");
180         if(src->motion_subsample_log2 != dst->motion_subsample_log2)
181             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_subsample_log2 doesn't match! (%d!=%d)\n",
182             src->motion_subsample_log2, dst->motion_subsample_log2);
183
184         memcpy(dst->mb_type, src->mb_type, s->mb_stride * s->mb_height * sizeof(dst->mb_type[0]));
185
186         for(i=0; i<2; i++){
187             int stride= ((16*s->mb_width )>>src->motion_subsample_log2) + 1;
188             int height= ((16*s->mb_height)>>src->motion_subsample_log2);
189
190             if(src->motion_val[i] && src->motion_val[i] != dst->motion_val[i]){
191                 memcpy(dst->motion_val[i], src->motion_val[i], 2*stride*height*sizeof(int16_t));
192             }
193             if(src->ref_index[i] && src->ref_index[i] != dst->ref_index[i]){
194                 memcpy(dst->ref_index[i], src->ref_index[i], s->b8_stride*2*s->mb_height*sizeof(int8_t));
195             }
196         }
197     }
198 }
199
200 static void update_duplicate_context_after_me(MpegEncContext *dst, MpegEncContext *src){
201 #define COPY(a) dst->a= src->a
202     COPY(pict_type);
203     COPY(current_picture);
204     COPY(f_code);
205     COPY(b_code);
206     COPY(qscale);
207     COPY(lambda);
208     COPY(lambda2);
209     COPY(picture_in_gop_number);
210     COPY(gop_picture_number);
211     COPY(frame_pred_frame_dct); //FIXME don't set in encode_header
212     COPY(progressive_frame); //FIXME don't set in encode_header
213     COPY(partitioned_frame); //FIXME don't set in encode_header
214 #undef COPY
215 }
216
217 /**
218  * sets the given MpegEncContext to defaults for encoding.
219  * the changed fields will not depend upon the prior state of the MpegEncContext.
220  */
221 static void MPV_encode_defaults(MpegEncContext *s){
222     int i;
223     MPV_common_defaults(s);
224
225     for(i=-16; i<16; i++){
226         default_fcode_tab[i + MAX_MV]= 1;
227     }
228     s->me.mv_penalty= default_mv_penalty;
229     s->fcode_tab= default_fcode_tab;
230 }
231
232 /* init video encoder */
233 int MPV_encode_init(AVCodecContext *avctx)
234 {
235     MpegEncContext *s = avctx->priv_data;
236     int i;
237     int chroma_h_shift, chroma_v_shift;
238
239     MPV_encode_defaults(s);
240
241     switch (avctx->codec_id) {
242     case CODEC_ID_MPEG2VIDEO:
243         if(avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P){
244             av_log(avctx, AV_LOG_ERROR, "only YUV420 and YUV422 are supported\n");
245             return -1;
246         }
247         break;
248     case CODEC_ID_LJPEG:
249     case CODEC_ID_MJPEG:
250         if(avctx->pix_fmt != PIX_FMT_YUVJ420P && avctx->pix_fmt != PIX_FMT_YUVJ422P &&
251            ((avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P) || avctx->strict_std_compliance>FF_COMPLIANCE_INOFFICIAL)){
252             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
253             return -1;
254         }
255         break;
256     default:
257         if(avctx->pix_fmt != PIX_FMT_YUV420P){
258             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
259             return -1;
260         }
261     }
262
263     switch (avctx->pix_fmt) {
264     case PIX_FMT_YUVJ422P:
265     case PIX_FMT_YUV422P:
266         s->chroma_format = CHROMA_422;
267         break;
268     case PIX_FMT_YUVJ420P:
269     case PIX_FMT_YUV420P:
270     default:
271         s->chroma_format = CHROMA_420;
272         break;
273     }
274
275     s->bit_rate = avctx->bit_rate;
276     s->width = avctx->width;
277     s->height = avctx->height;
278     if(avctx->gop_size > 600 && avctx->strict_std_compliance>FF_COMPLIANCE_EXPERIMENTAL){
279         av_log(avctx, AV_LOG_ERROR, "Warning keyframe interval too large! reducing it ...\n");
280         avctx->gop_size=600;
281     }
282     s->gop_size = avctx->gop_size;
283     s->avctx = avctx;
284     s->flags= avctx->flags;
285     s->flags2= avctx->flags2;
286     s->max_b_frames= avctx->max_b_frames;
287     s->codec_id= avctx->codec->id;
288     s->luma_elim_threshold  = avctx->luma_elim_threshold;
289     s->chroma_elim_threshold= avctx->chroma_elim_threshold;
290     s->strict_std_compliance= avctx->strict_std_compliance;
291     s->data_partitioning= avctx->flags & CODEC_FLAG_PART;
292     s->quarter_sample= (avctx->flags & CODEC_FLAG_QPEL)!=0;
293     s->mpeg_quant= avctx->mpeg_quant;
294     s->rtp_mode= !!avctx->rtp_payload_size;
295     s->intra_dc_precision= avctx->intra_dc_precision;
296     s->user_specified_pts = AV_NOPTS_VALUE;
297
298     if (s->gop_size <= 1) {
299         s->intra_only = 1;
300         s->gop_size = 12;
301     } else {
302         s->intra_only = 0;
303     }
304
305     s->me_method = avctx->me_method;
306
307     /* Fixed QSCALE */
308     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
309
310     s->adaptive_quant= (   s->avctx->lumi_masking
311                         || s->avctx->dark_masking
312                         || s->avctx->temporal_cplx_masking
313                         || s->avctx->spatial_cplx_masking
314                         || s->avctx->p_masking
315                         || s->avctx->border_masking
316                         || (s->flags&CODEC_FLAG_QP_RD))
317                        && !s->fixed_qscale;
318
319     s->obmc= !!(s->flags & CODEC_FLAG_OBMC);
320     s->loop_filter= !!(s->flags & CODEC_FLAG_LOOP_FILTER);
321     s->alternate_scan= !!(s->flags & CODEC_FLAG_ALT_SCAN);
322     s->intra_vlc_format= !!(s->flags2 & CODEC_FLAG2_INTRA_VLC);
323     s->q_scale_type= !!(s->flags2 & CODEC_FLAG2_NON_LINEAR_QUANT);
324
325     if(avctx->rc_max_rate && !avctx->rc_buffer_size){
326         av_log(avctx, AV_LOG_ERROR, "a vbv buffer size is needed, for encoding with a maximum bitrate\n");
327         return -1;
328     }
329
330     if(avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate){
331         av_log(avctx, AV_LOG_INFO, "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
332     }
333
334     if(avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate){
335         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
336         return -1;
337     }
338
339     if(avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate){
340         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
341         return -1;
342     }
343
344     if(avctx->rc_buffer_size && avctx->bit_rate*av_q2d(avctx->time_base) > avctx->rc_buffer_size){
345         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
346         return -1;
347     }
348
349     if(avctx->bit_rate*av_q2d(avctx->time_base) > avctx->bit_rate_tolerance){
350         av_log(avctx, AV_LOG_ERROR, "bitrate tolerance too small for bitrate\n");
351         return -1;
352     }
353
354     if(   s->avctx->rc_max_rate && s->avctx->rc_min_rate == s->avctx->rc_max_rate
355        && (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO)
356        && 90000LL * (avctx->rc_buffer_size-1) > s->avctx->rc_max_rate*0xFFFFLL){
357
358         av_log(avctx, AV_LOG_INFO, "Warning vbv_delay will be set to 0xFFFF (=VBR) as the specified vbv buffer is too large for the given bitrate!\n");
359     }
360
361     if((s->flags & CODEC_FLAG_4MV) && s->codec_id != CODEC_ID_MPEG4
362        && s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P && s->codec_id != CODEC_ID_FLV1){
363         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
364         return -1;
365     }
366
367     if(s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE){
368         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with simple mb decision\n");
369         return -1;
370     }
371
372     if(s->obmc && s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P){
373         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with H263(+)\n");
374         return -1;
375     }
376
377     if(s->quarter_sample && s->codec_id != CODEC_ID_MPEG4){
378         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
379         return -1;
380     }
381
382     if(s->data_partitioning && s->codec_id != CODEC_ID_MPEG4){
383         av_log(avctx, AV_LOG_ERROR, "data partitioning not supported by codec\n");
384         return -1;
385     }
386
387     if(s->max_b_frames && s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG1VIDEO && s->codec_id != CODEC_ID_MPEG2VIDEO){
388         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
389         return -1;
390     }
391
392     if((s->flags & (CODEC_FLAG_INTERLACED_DCT|CODEC_FLAG_INTERLACED_ME|CODEC_FLAG_ALT_SCAN))
393        && s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG2VIDEO){
394         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
395         return -1;
396     }
397
398     if(s->mpeg_quant && s->codec_id != CODEC_ID_MPEG4){ //FIXME mpeg2 uses that too
399         av_log(avctx, AV_LOG_ERROR, "mpeg2 style quantization not supported by codec\n");
400         return -1;
401     }
402
403     if((s->flags & CODEC_FLAG_CBP_RD) && !(s->flags & CODEC_FLAG_TRELLIS_QUANT)){
404         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
405         return -1;
406     }
407
408     if((s->flags & CODEC_FLAG_QP_RD) && s->avctx->mb_decision != FF_MB_DECISION_RD){
409         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
410         return -1;
411     }
412
413     if(s->avctx->scenechange_threshold < 1000000000 && (s->flags & CODEC_FLAG_CLOSED_GOP)){
414         av_log(avctx, AV_LOG_ERROR, "closed gop with scene change detection arent supported yet, set threshold to 1000000000\n");
415         return -1;
416     }
417
418     if((s->flags2 & CODEC_FLAG2_INTRA_VLC) && s->codec_id != CODEC_ID_MPEG2VIDEO){
419         av_log(avctx, AV_LOG_ERROR, "intra vlc table not supported by codec\n");
420         return -1;
421     }
422
423     if(s->flags & CODEC_FLAG_LOW_DELAY){
424         if (s->codec_id != CODEC_ID_MPEG2VIDEO && s->codec_id != CODEC_ID_MPEG1VIDEO){
425             av_log(avctx, AV_LOG_ERROR, "low delay forcing is only available for mpeg1/2\n");
426             return -1;
427         }
428         if (s->max_b_frames != 0){
429             av_log(avctx, AV_LOG_ERROR, "b frames cannot be used with low delay\n");
430             return -1;
431         }
432     }
433
434     if(s->q_scale_type == 1){
435         if(s->codec_id != CODEC_ID_MPEG2VIDEO){
436             av_log(avctx, AV_LOG_ERROR, "non linear quant is only available for mpeg2\n");
437             return -1;
438         }
439         if(avctx->qmax > 12){
440             av_log(avctx, AV_LOG_ERROR, "non linear quant only supports qmax <= 12 currently\n");
441             return -1;
442         }
443     }
444
445     if(s->avctx->thread_count > 1 && s->codec_id != CODEC_ID_MPEG4
446        && s->codec_id != CODEC_ID_MPEG1VIDEO && s->codec_id != CODEC_ID_MPEG2VIDEO
447        && (s->codec_id != CODEC_ID_H263P || !(s->flags & CODEC_FLAG_H263P_SLICE_STRUCT))){
448         av_log(avctx, AV_LOG_ERROR, "multi threaded encoding not supported by codec\n");
449         return -1;
450     }
451
452     if(s->avctx->thread_count > 1)
453         s->rtp_mode= 1;
454
455     if(!avctx->time_base.den || !avctx->time_base.num){
456         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
457         return -1;
458     }
459
460     i= (INT_MAX/2+128)>>8;
461     if(avctx->me_threshold >= i){
462         av_log(avctx, AV_LOG_ERROR, "me_threshold too large, max is %d\n", i - 1);
463         return -1;
464     }
465     if(avctx->mb_threshold >= i){
466         av_log(avctx, AV_LOG_ERROR, "mb_threshold too large, max is %d\n", i - 1);
467         return -1;
468     }
469
470     if(avctx->b_frame_strategy && (avctx->flags&CODEC_FLAG_PASS2)){
471         av_log(avctx, AV_LOG_INFO, "notice: b_frame_strategy only affects the first pass\n");
472         avctx->b_frame_strategy = 0;
473     }
474
475     i= ff_gcd(avctx->time_base.den, avctx->time_base.num);
476     if(i > 1){
477         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
478         avctx->time_base.den /= i;
479         avctx->time_base.num /= i;
480 //        return -1;
481     }
482
483     if(s->codec_id==CODEC_ID_MJPEG){
484         s->intra_quant_bias= 1<<(QUANT_BIAS_SHIFT-1); //(a + x/2)/x
485         s->inter_quant_bias= 0;
486     }else if(s->mpeg_quant || s->codec_id==CODEC_ID_MPEG1VIDEO || s->codec_id==CODEC_ID_MPEG2VIDEO){
487         s->intra_quant_bias= 3<<(QUANT_BIAS_SHIFT-3); //(a + x*3/8)/x
488         s->inter_quant_bias= 0;
489     }else{
490         s->intra_quant_bias=0;
491         s->inter_quant_bias=-(1<<(QUANT_BIAS_SHIFT-2)); //(a - x/4)/x
492     }
493
494     if(avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
495         s->intra_quant_bias= avctx->intra_quant_bias;
496     if(avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
497         s->inter_quant_bias= avctx->inter_quant_bias;
498
499     avcodec_get_chroma_sub_sample(avctx->pix_fmt, &chroma_h_shift, &chroma_v_shift);
500
501     if(avctx->codec_id == CODEC_ID_MPEG4 && s->avctx->time_base.den > (1<<16)-1){
502         av_log(avctx, AV_LOG_ERROR, "timebase not supported by mpeg 4 standard\n");
503         return -1;
504     }
505     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
506
507     switch(avctx->codec->id) {
508     case CODEC_ID_MPEG1VIDEO:
509         s->out_format = FMT_MPEG1;
510         s->low_delay= !!(s->flags & CODEC_FLAG_LOW_DELAY);
511         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
512         break;
513     case CODEC_ID_MPEG2VIDEO:
514         s->out_format = FMT_MPEG1;
515         s->low_delay= !!(s->flags & CODEC_FLAG_LOW_DELAY);
516         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
517         s->rtp_mode= 1;
518         break;
519     case CODEC_ID_LJPEG:
520     case CODEC_ID_MJPEG:
521         s->out_format = FMT_MJPEG;
522         s->intra_only = 1; /* force intra only for jpeg */
523         s->mjpeg_vsample[0] = 2;
524         s->mjpeg_vsample[1] = 2>>chroma_v_shift;
525         s->mjpeg_vsample[2] = 2>>chroma_v_shift;
526         s->mjpeg_hsample[0] = 2;
527         s->mjpeg_hsample[1] = 2>>chroma_h_shift;
528         s->mjpeg_hsample[2] = 2>>chroma_h_shift;
529         if (!(ENABLE_MJPEG_ENCODER || ENABLE_LJPEG_ENCODER)
530             || ff_mjpeg_encode_init(s) < 0)
531             return -1;
532         avctx->delay=0;
533         s->low_delay=1;
534         break;
535     case CODEC_ID_H261:
536         if (!ENABLE_H261_ENCODER)  return -1;
537         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
538             av_log(avctx, AV_LOG_ERROR, "The specified picture size of %dx%d is not valid for the H.261 codec.\nValid sizes are 176x144, 352x288\n", s->width, s->height);
539             return -1;
540         }
541         s->out_format = FMT_H261;
542         avctx->delay=0;
543         s->low_delay=1;
544         break;
545     case CODEC_ID_H263:
546         if (h263_get_picture_format(s->width, s->height) == 7) {
547             av_log(avctx, AV_LOG_INFO, "The specified picture size of %dx%d is not valid for the H.263 codec.\nValid sizes are 128x96, 176x144, 352x288, 704x576, and 1408x1152. Try H.263+.\n", s->width, s->height);
548             return -1;
549         }
550         s->out_format = FMT_H263;
551         s->obmc= (avctx->flags & CODEC_FLAG_OBMC) ? 1:0;
552         avctx->delay=0;
553         s->low_delay=1;
554         break;
555     case CODEC_ID_H263P:
556         s->out_format = FMT_H263;
557         s->h263_plus = 1;
558         /* Fx */
559         s->umvplus = (avctx->flags & CODEC_FLAG_H263P_UMV) ? 1:0;
560         s->h263_aic= (avctx->flags & CODEC_FLAG_AC_PRED) ? 1:0;
561         s->modified_quant= s->h263_aic;
562         s->alt_inter_vlc= (avctx->flags & CODEC_FLAG_H263P_AIV) ? 1:0;
563         s->obmc= (avctx->flags & CODEC_FLAG_OBMC) ? 1:0;
564         s->loop_filter= (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1:0;
565         s->unrestricted_mv= s->obmc || s->loop_filter || s->umvplus;
566         s->h263_slice_structured= (s->flags & CODEC_FLAG_H263P_SLICE_STRUCT) ? 1:0;
567
568         /* /Fx */
569         /* These are just to be sure */
570         avctx->delay=0;
571         s->low_delay=1;
572         break;
573     case CODEC_ID_FLV1:
574         s->out_format = FMT_H263;
575         s->h263_flv = 2; /* format = 1; 11-bit codes */
576         s->unrestricted_mv = 1;
577         s->rtp_mode=0; /* don't allow GOB */
578         avctx->delay=0;
579         s->low_delay=1;
580         break;
581     case CODEC_ID_RV10:
582         s->out_format = FMT_H263;
583         avctx->delay=0;
584         s->low_delay=1;
585         break;
586     case CODEC_ID_RV20:
587         s->out_format = FMT_H263;
588         avctx->delay=0;
589         s->low_delay=1;
590         s->modified_quant=1;
591         s->h263_aic=1;
592         s->h263_plus=1;
593         s->loop_filter=1;
594         s->unrestricted_mv= s->obmc || s->loop_filter || s->umvplus;
595         break;
596     case CODEC_ID_MPEG4:
597         s->out_format = FMT_H263;
598         s->h263_pred = 1;
599         s->unrestricted_mv = 1;
600         s->low_delay= s->max_b_frames ? 0 : 1;
601         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
602         break;
603     case CODEC_ID_MSMPEG4V1:
604         s->out_format = FMT_H263;
605         s->h263_msmpeg4 = 1;
606         s->h263_pred = 1;
607         s->unrestricted_mv = 1;
608         s->msmpeg4_version= 1;
609         avctx->delay=0;
610         s->low_delay=1;
611         break;
612     case CODEC_ID_MSMPEG4V2:
613         s->out_format = FMT_H263;
614         s->h263_msmpeg4 = 1;
615         s->h263_pred = 1;
616         s->unrestricted_mv = 1;
617         s->msmpeg4_version= 2;
618         avctx->delay=0;
619         s->low_delay=1;
620         break;
621     case CODEC_ID_MSMPEG4V3:
622         s->out_format = FMT_H263;
623         s->h263_msmpeg4 = 1;
624         s->h263_pred = 1;
625         s->unrestricted_mv = 1;
626         s->msmpeg4_version= 3;
627         s->flipflop_rounding=1;
628         avctx->delay=0;
629         s->low_delay=1;
630         break;
631     case CODEC_ID_WMV1:
632         s->out_format = FMT_H263;
633         s->h263_msmpeg4 = 1;
634         s->h263_pred = 1;
635         s->unrestricted_mv = 1;
636         s->msmpeg4_version= 4;
637         s->flipflop_rounding=1;
638         avctx->delay=0;
639         s->low_delay=1;
640         break;
641     case CODEC_ID_WMV2:
642         s->out_format = FMT_H263;
643         s->h263_msmpeg4 = 1;
644         s->h263_pred = 1;
645         s->unrestricted_mv = 1;
646         s->msmpeg4_version= 5;
647         s->flipflop_rounding=1;
648         avctx->delay=0;
649         s->low_delay=1;
650         break;
651     default:
652         return -1;
653     }
654
655     avctx->has_b_frames= !s->low_delay;
656
657     s->encoding = 1;
658
659     /* init */
660     if (MPV_common_init(s) < 0)
661         return -1;
662
663     if(!s->dct_quantize)
664         s->dct_quantize = dct_quantize_c;
665     if(!s->denoise_dct)
666         s->denoise_dct = denoise_dct_c;
667     s->fast_dct_quantize = s->dct_quantize;
668     if(s->flags & CODEC_FLAG_TRELLIS_QUANT)
669         s->dct_quantize = dct_quantize_trellis_c;
670
671     if(s->modified_quant)
672         s->chroma_qscale_table= ff_h263_chroma_qscale_table;
673     s->progressive_frame=
674     s->progressive_sequence= !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT|CODEC_FLAG_INTERLACED_ME|CODEC_FLAG_ALT_SCAN));
675     s->quant_precision=5;
676
677     ff_set_cmp(&s->dsp, s->dsp.ildct_cmp, s->avctx->ildct_cmp);
678     ff_set_cmp(&s->dsp, s->dsp.frame_skip_cmp, s->avctx->frame_skip_cmp);
679
680     if (ENABLE_H261_ENCODER && s->out_format == FMT_H261)
681         ff_h261_encode_init(s);
682     if (s->out_format == FMT_H263)
683         h263_encode_init(s);
684     if (ENABLE_MSMPEG4_ENCODER && s->msmpeg4_version)
685         ff_msmpeg4_encode_init(s);
686     if ((ENABLE_MPEG1VIDEO_ENCODER || ENABLE_MPEG2VIDEO_ENCODER)
687         && s->out_format == FMT_MPEG1)
688         ff_mpeg1_encode_init(s);
689
690     /* init q matrix */
691     for(i=0;i<64;i++) {
692         int j= s->dsp.idct_permutation[i];
693         if(s->codec_id==CODEC_ID_MPEG4 && s->mpeg_quant){
694             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
695             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
696         }else if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
697             s->intra_matrix[j] =
698             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
699         }else
700         { /* mpeg1/2 */
701             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
702             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
703         }
704         if(s->avctx->intra_matrix)
705             s->intra_matrix[j] = s->avctx->intra_matrix[i];
706         if(s->avctx->inter_matrix)
707             s->inter_matrix[j] = s->avctx->inter_matrix[i];
708     }
709
710     /* precompute matrix */
711     /* for mjpeg, we do include qscale in the matrix */
712     if (s->out_format != FMT_MJPEG) {
713         convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
714                        s->intra_matrix, s->intra_quant_bias, avctx->qmin, 31, 1);
715         convert_matrix(&s->dsp, s->q_inter_matrix, s->q_inter_matrix16,
716                        s->inter_matrix, s->inter_quant_bias, avctx->qmin, 31, 0);
717     }
718
719     if(ff_rate_control_init(s) < 0)
720         return -1;
721
722     return 0;
723 }
724
725 int MPV_encode_end(AVCodecContext *avctx)
726 {
727     MpegEncContext *s = avctx->priv_data;
728
729     ff_rate_control_uninit(s);
730
731     MPV_common_end(s);
732     if ((ENABLE_MJPEG_ENCODER || ENABLE_LJPEG_ENCODER) && s->out_format == FMT_MJPEG)
733         ff_mjpeg_encode_close(s);
734
735     av_freep(&avctx->extradata);
736
737     return 0;
738 }
739
740 static int get_sae(uint8_t *src, int ref, int stride){
741     int x,y;
742     int acc=0;
743
744     for(y=0; y<16; y++){
745         for(x=0; x<16; x++){
746             acc+= FFABS(src[x+y*stride] - ref);
747         }
748     }
749
750     return acc;
751 }
752
753 static int get_intra_count(MpegEncContext *s, uint8_t *src, uint8_t *ref, int stride){
754     int x, y, w, h;
755     int acc=0;
756
757     w= s->width &~15;
758     h= s->height&~15;
759
760     for(y=0; y<h; y+=16){
761         for(x=0; x<w; x+=16){
762             int offset= x + y*stride;
763             int sad = s->dsp.sad[0](NULL, src + offset, ref + offset, stride, 16);
764             int mean= (s->dsp.pix_sum(src + offset, stride) + 128)>>8;
765             int sae = get_sae(src + offset, mean, stride);
766
767             acc+= sae + 500 < sad;
768         }
769     }
770     return acc;
771 }
772
773
774 static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg){
775     AVFrame *pic=NULL;
776     int64_t pts;
777     int i;
778     const int encoding_delay= s->max_b_frames;
779     int direct=1;
780
781     if(pic_arg){
782         pts= pic_arg->pts;
783         pic_arg->display_picture_number= s->input_picture_number++;
784
785         if(pts != AV_NOPTS_VALUE){
786             if(s->user_specified_pts != AV_NOPTS_VALUE){
787                 int64_t time= pts;
788                 int64_t last= s->user_specified_pts;
789
790                 if(time <= last){
791                     av_log(s->avctx, AV_LOG_ERROR, "Error, Invalid timestamp=%"PRId64", last=%"PRId64"\n", pts, s->user_specified_pts);
792                     return -1;
793                 }
794             }
795             s->user_specified_pts= pts;
796         }else{
797             if(s->user_specified_pts != AV_NOPTS_VALUE){
798                 s->user_specified_pts=
799                 pts= s->user_specified_pts + 1;
800                 av_log(s->avctx, AV_LOG_INFO, "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n", pts);
801             }else{
802                 pts= pic_arg->display_picture_number;
803             }
804         }
805     }
806
807   if(pic_arg){
808     if(encoding_delay && !(s->flags&CODEC_FLAG_INPUT_PRESERVED)) direct=0;
809     if(pic_arg->linesize[0] != s->linesize) direct=0;
810     if(pic_arg->linesize[1] != s->uvlinesize) direct=0;
811     if(pic_arg->linesize[2] != s->uvlinesize) direct=0;
812
813 //    av_log(AV_LOG_DEBUG, "%d %d %d %d\n",pic_arg->linesize[0], pic_arg->linesize[1], s->linesize, s->uvlinesize);
814
815     if(direct){
816         i= ff_find_unused_picture(s, 1);
817
818         pic= (AVFrame*)&s->picture[i];
819         pic->reference= 3;
820
821         for(i=0; i<4; i++){
822             pic->data[i]= pic_arg->data[i];
823             pic->linesize[i]= pic_arg->linesize[i];
824         }
825         alloc_picture(s, (Picture*)pic, 1);
826     }else{
827         i= ff_find_unused_picture(s, 0);
828
829         pic= (AVFrame*)&s->picture[i];
830         pic->reference= 3;
831
832         alloc_picture(s, (Picture*)pic, 0);
833
834         if(   pic->data[0] + INPLACE_OFFSET == pic_arg->data[0]
835            && pic->data[1] + INPLACE_OFFSET == pic_arg->data[1]
836            && pic->data[2] + INPLACE_OFFSET == pic_arg->data[2]){
837        // empty
838         }else{
839             int h_chroma_shift, v_chroma_shift;
840             avcodec_get_chroma_sub_sample(s->avctx->pix_fmt, &h_chroma_shift, &v_chroma_shift);
841
842             for(i=0; i<3; i++){
843                 int src_stride= pic_arg->linesize[i];
844                 int dst_stride= i ? s->uvlinesize : s->linesize;
845                 int h_shift= i ? h_chroma_shift : 0;
846                 int v_shift= i ? v_chroma_shift : 0;
847                 int w= s->width >>h_shift;
848                 int h= s->height>>v_shift;
849                 uint8_t *src= pic_arg->data[i];
850                 uint8_t *dst= pic->data[i];
851
852                 if(!s->avctx->rc_buffer_size)
853                     dst +=INPLACE_OFFSET;
854
855                 if(src_stride==dst_stride)
856                     memcpy(dst, src, src_stride*h);
857                 else{
858                     while(h--){
859                         memcpy(dst, src, w);
860                         dst += dst_stride;
861                         src += src_stride;
862                     }
863                 }
864             }
865         }
866     }
867     copy_picture_attributes(s, pic, pic_arg);
868     pic->pts= pts; //we set this here to avoid modifiying pic_arg
869   }
870
871     /* shift buffer entries */
872     for(i=1; i<MAX_PICTURE_COUNT /*s->encoding_delay+1*/; i++)
873         s->input_picture[i-1]= s->input_picture[i];
874
875     s->input_picture[encoding_delay]= (Picture*)pic;
876
877     return 0;
878 }
879
880 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref){
881     int x, y, plane;
882     int score=0;
883     int64_t score64=0;
884
885     for(plane=0; plane<3; plane++){
886         const int stride= p->linesize[plane];
887         const int bw= plane ? 1 : 2;
888         for(y=0; y<s->mb_height*bw; y++){
889             for(x=0; x<s->mb_width*bw; x++){
890                 int off= p->type == FF_BUFFER_TYPE_SHARED ? 0: 16;
891                 int v= s->dsp.frame_skip_cmp[1](s, p->data[plane] + 8*(x + y*stride)+off, ref->data[plane] + 8*(x + y*stride), stride, 8);
892
893                 switch(s->avctx->frame_skip_exp){
894                     case 0: score= FFMAX(score, v); break;
895                     case 1: score+= FFABS(v);break;
896                     case 2: score+= v*v;break;
897                     case 3: score64+= FFABS(v*v*(int64_t)v);break;
898                     case 4: score64+= v*v*(int64_t)(v*v);break;
899                 }
900             }
901         }
902     }
903
904     if(score) score64= score;
905
906     if(score64 < s->avctx->frame_skip_threshold)
907         return 1;
908     if(score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda)>>8))
909         return 1;
910     return 0;
911 }
912
913 static int estimate_best_b_count(MpegEncContext *s){
914     AVCodec *codec= avcodec_find_encoder(s->avctx->codec_id);
915     AVCodecContext *c= avcodec_alloc_context();
916     AVFrame input[FF_MAX_B_FRAMES+2];
917     const int scale= s->avctx->brd_scale;
918     int i, j, out_size, p_lambda, b_lambda, lambda2;
919     int outbuf_size= s->width * s->height; //FIXME
920     uint8_t *outbuf= av_malloc(outbuf_size);
921     int64_t best_rd= INT64_MAX;
922     int best_b_count= -1;
923
924     assert(scale>=0 && scale <=3);
925
926 //    emms_c();
927     p_lambda= s->last_lambda_for[P_TYPE]; //s->next_picture_ptr->quality;
928     b_lambda= s->last_lambda_for[B_TYPE]; //p_lambda *FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
929     if(!b_lambda) b_lambda= p_lambda; //FIXME we should do this somewhere else
930     lambda2= (b_lambda*b_lambda + (1<<FF_LAMBDA_SHIFT)/2 ) >> FF_LAMBDA_SHIFT;
931
932     c->width = s->width >> scale;
933     c->height= s->height>> scale;
934     c->flags= CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR | CODEC_FLAG_INPUT_PRESERVED /*| CODEC_FLAG_EMU_EDGE*/;
935     c->flags|= s->avctx->flags & CODEC_FLAG_QPEL;
936     c->mb_decision= s->avctx->mb_decision;
937     c->me_cmp= s->avctx->me_cmp;
938     c->mb_cmp= s->avctx->mb_cmp;
939     c->me_sub_cmp= s->avctx->me_sub_cmp;
940     c->pix_fmt = PIX_FMT_YUV420P;
941     c->time_base= s->avctx->time_base;
942     c->max_b_frames= s->max_b_frames;
943
944     if (avcodec_open(c, codec) < 0)
945         return -1;
946
947     for(i=0; i<s->max_b_frames+2; i++){
948         int ysize= c->width*c->height;
949         int csize= (c->width/2)*(c->height/2);
950         Picture pre_input, *pre_input_ptr= i ? s->input_picture[i-1] : s->next_picture_ptr;
951
952         avcodec_get_frame_defaults(&input[i]);
953         input[i].data[0]= av_malloc(ysize + 2*csize);
954         input[i].data[1]= input[i].data[0] + ysize;
955         input[i].data[2]= input[i].data[1] + csize;
956         input[i].linesize[0]= c->width;
957         input[i].linesize[1]=
958         input[i].linesize[2]= c->width/2;
959
960         if(pre_input_ptr && (!i || s->input_picture[i-1])) {
961             pre_input= *pre_input_ptr;
962
963             if(pre_input.type != FF_BUFFER_TYPE_SHARED && i) {
964                 pre_input.data[0]+=INPLACE_OFFSET;
965                 pre_input.data[1]+=INPLACE_OFFSET;
966                 pre_input.data[2]+=INPLACE_OFFSET;
967             }
968
969             s->dsp.shrink[scale](input[i].data[0], input[i].linesize[0], pre_input.data[0], pre_input.linesize[0], c->width, c->height);
970             s->dsp.shrink[scale](input[i].data[1], input[i].linesize[1], pre_input.data[1], pre_input.linesize[1], c->width>>1, c->height>>1);
971             s->dsp.shrink[scale](input[i].data[2], input[i].linesize[2], pre_input.data[2], pre_input.linesize[2], c->width>>1, c->height>>1);
972         }
973     }
974
975     for(j=0; j<s->max_b_frames+1; j++){
976         int64_t rd=0;
977
978         if(!s->input_picture[j])
979             break;
980
981         c->error[0]= c->error[1]= c->error[2]= 0;
982
983         input[0].pict_type= I_TYPE;
984         input[0].quality= 1 * FF_QP2LAMBDA;
985         out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[0]);
986 //        rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
987
988         for(i=0; i<s->max_b_frames+1; i++){
989             int is_p= i % (j+1) == j || i==s->max_b_frames;
990
991             input[i+1].pict_type= is_p ? P_TYPE : B_TYPE;
992             input[i+1].quality= is_p ? p_lambda : b_lambda;
993             out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[i+1]);
994             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
995         }
996
997         /* get the delayed frames */
998         while(out_size){
999             out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL);
1000             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1001         }
1002
1003         rd += c->error[0] + c->error[1] + c->error[2];
1004
1005         if(rd < best_rd){
1006             best_rd= rd;
1007             best_b_count= j;
1008         }
1009     }
1010
1011     av_freep(&outbuf);
1012     avcodec_close(c);
1013     av_freep(&c);
1014
1015     for(i=0; i<s->max_b_frames+2; i++){
1016         av_freep(&input[i].data[0]);
1017     }
1018
1019     return best_b_count;
1020 }
1021
1022 static void select_input_picture(MpegEncContext *s){
1023     int i;
1024
1025     for(i=1; i<MAX_PICTURE_COUNT; i++)
1026         s->reordered_input_picture[i-1]= s->reordered_input_picture[i];
1027     s->reordered_input_picture[MAX_PICTURE_COUNT-1]= NULL;
1028
1029     /* set next picture type & ordering */
1030     if(s->reordered_input_picture[0]==NULL && s->input_picture[0]){
1031         if(/*s->picture_in_gop_number >= s->gop_size ||*/ s->next_picture_ptr==NULL || s->intra_only){
1032             s->reordered_input_picture[0]= s->input_picture[0];
1033             s->reordered_input_picture[0]->pict_type= I_TYPE;
1034             s->reordered_input_picture[0]->coded_picture_number= s->coded_picture_number++;
1035         }else{
1036             int b_frames;
1037
1038             if(s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor){
1039                 if(s->picture_in_gop_number < s->gop_size && skip_check(s, s->input_picture[0], s->next_picture_ptr)){
1040                 //FIXME check that te gop check above is +-1 correct
1041 //av_log(NULL, AV_LOG_DEBUG, "skip %p %"PRId64"\n", s->input_picture[0]->data[0], s->input_picture[0]->pts);
1042
1043                     if(s->input_picture[0]->type == FF_BUFFER_TYPE_SHARED){
1044                         for(i=0; i<4; i++)
1045                             s->input_picture[0]->data[i]= NULL;
1046                         s->input_picture[0]->type= 0;
1047                     }else{
1048                         assert(   s->input_picture[0]->type==FF_BUFFER_TYPE_USER
1049                                || s->input_picture[0]->type==FF_BUFFER_TYPE_INTERNAL);
1050
1051                         s->avctx->release_buffer(s->avctx, (AVFrame*)s->input_picture[0]);
1052                     }
1053
1054                     emms_c();
1055                     ff_vbv_update(s, 0);
1056
1057                     goto no_output_pic;
1058                 }
1059             }
1060
1061             if(s->flags&CODEC_FLAG_PASS2){
1062                 for(i=0; i<s->max_b_frames+1; i++){
1063                     int pict_num= s->input_picture[0]->display_picture_number + i;
1064
1065                     if(pict_num >= s->rc_context.num_entries)
1066                         break;
1067                     if(!s->input_picture[i]){
1068                         s->rc_context.entry[pict_num-1].new_pict_type = P_TYPE;
1069                         break;
1070                     }
1071
1072                     s->input_picture[i]->pict_type=
1073                         s->rc_context.entry[pict_num].new_pict_type;
1074                 }
1075             }
1076
1077             if(s->avctx->b_frame_strategy==0){
1078                 b_frames= s->max_b_frames;
1079                 while(b_frames && !s->input_picture[b_frames]) b_frames--;
1080             }else if(s->avctx->b_frame_strategy==1){
1081                 for(i=1; i<s->max_b_frames+1; i++){
1082                     if(s->input_picture[i] && s->input_picture[i]->b_frame_score==0){
1083                         s->input_picture[i]->b_frame_score=
1084                             get_intra_count(s, s->input_picture[i  ]->data[0],
1085                                                s->input_picture[i-1]->data[0], s->linesize) + 1;
1086                     }
1087                 }
1088                 for(i=0; i<s->max_b_frames+1; i++){
1089                     if(s->input_picture[i]==NULL || s->input_picture[i]->b_frame_score - 1 > s->mb_num/s->avctx->b_sensitivity) break;
1090                 }
1091
1092                 b_frames= FFMAX(0, i-1);
1093
1094                 /* reset scores */
1095                 for(i=0; i<b_frames+1; i++){
1096                     s->input_picture[i]->b_frame_score=0;
1097                 }
1098             }else if(s->avctx->b_frame_strategy==2){
1099                 b_frames= estimate_best_b_count(s);
1100             }else{
1101                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1102                 b_frames=0;
1103             }
1104
1105             emms_c();
1106 //static int b_count=0;
1107 //b_count+= b_frames;
1108 //av_log(s->avctx, AV_LOG_DEBUG, "b_frames: %d\n", b_count);
1109
1110             for(i= b_frames - 1; i>=0; i--){
1111                 int type= s->input_picture[i]->pict_type;
1112                 if(type && type != B_TYPE)
1113                     b_frames= i;
1114             }
1115             if(s->input_picture[b_frames]->pict_type == B_TYPE && b_frames == s->max_b_frames){
1116                 av_log(s->avctx, AV_LOG_ERROR, "warning, too many b frames in a row\n");
1117             }
1118
1119             if(s->picture_in_gop_number + b_frames >= s->gop_size){
1120               if((s->flags2 & CODEC_FLAG2_STRICT_GOP) && s->gop_size > s->picture_in_gop_number){
1121                     b_frames= s->gop_size - s->picture_in_gop_number - 1;
1122               }else{
1123                 if(s->flags & CODEC_FLAG_CLOSED_GOP)
1124                     b_frames=0;
1125                 s->input_picture[b_frames]->pict_type= I_TYPE;
1126               }
1127             }
1128
1129             if(   (s->flags & CODEC_FLAG_CLOSED_GOP)
1130                && b_frames
1131                && s->input_picture[b_frames]->pict_type== I_TYPE)
1132                 b_frames--;
1133
1134             s->reordered_input_picture[0]= s->input_picture[b_frames];
1135             if(s->reordered_input_picture[0]->pict_type != I_TYPE)
1136                 s->reordered_input_picture[0]->pict_type= P_TYPE;
1137             s->reordered_input_picture[0]->coded_picture_number= s->coded_picture_number++;
1138             for(i=0; i<b_frames; i++){
1139                 s->reordered_input_picture[i+1]= s->input_picture[i];
1140                 s->reordered_input_picture[i+1]->pict_type= B_TYPE;
1141                 s->reordered_input_picture[i+1]->coded_picture_number= s->coded_picture_number++;
1142             }
1143         }
1144     }
1145 no_output_pic:
1146     if(s->reordered_input_picture[0]){
1147         s->reordered_input_picture[0]->reference= s->reordered_input_picture[0]->pict_type!=B_TYPE ? 3 : 0;
1148
1149         copy_picture(&s->new_picture, s->reordered_input_picture[0]);
1150
1151         if(s->reordered_input_picture[0]->type == FF_BUFFER_TYPE_SHARED || s->avctx->rc_buffer_size){
1152             // input is a shared pix, so we can't modifiy it -> alloc a new one & ensure that the shared one is reuseable
1153
1154             int i= ff_find_unused_picture(s, 0);
1155             Picture *pic= &s->picture[i];
1156
1157             pic->reference              = s->reordered_input_picture[0]->reference;
1158             alloc_picture(s, pic, 0);
1159
1160             /* mark us unused / free shared pic */
1161             if(s->reordered_input_picture[0]->type == FF_BUFFER_TYPE_INTERNAL)
1162                 s->avctx->release_buffer(s->avctx, (AVFrame*)s->reordered_input_picture[0]);
1163             for(i=0; i<4; i++)
1164                 s->reordered_input_picture[0]->data[i]= NULL;
1165             s->reordered_input_picture[0]->type= 0;
1166
1167             copy_picture_attributes(s, (AVFrame*)pic, (AVFrame*)s->reordered_input_picture[0]);
1168
1169             s->current_picture_ptr= pic;
1170         }else{
1171             // input is not a shared pix -> reuse buffer for current_pix
1172
1173             assert(   s->reordered_input_picture[0]->type==FF_BUFFER_TYPE_USER
1174                    || s->reordered_input_picture[0]->type==FF_BUFFER_TYPE_INTERNAL);
1175
1176             s->current_picture_ptr= s->reordered_input_picture[0];
1177             for(i=0; i<4; i++){
1178                 s->new_picture.data[i]+= INPLACE_OFFSET;
1179             }
1180         }
1181         copy_picture(&s->current_picture, s->current_picture_ptr);
1182
1183         s->picture_number= s->new_picture.display_picture_number;
1184 //printf("dpn:%d\n", s->picture_number);
1185     }else{
1186        memset(&s->new_picture, 0, sizeof(Picture));
1187     }
1188 }
1189
1190 int MPV_encode_picture(AVCodecContext *avctx,
1191                        unsigned char *buf, int buf_size, void *data)
1192 {
1193     MpegEncContext *s = avctx->priv_data;
1194     AVFrame *pic_arg = data;
1195     int i, stuffing_count;
1196
1197     for(i=0; i<avctx->thread_count; i++){
1198         int start_y= s->thread_context[i]->start_mb_y;
1199         int   end_y= s->thread_context[i]->  end_mb_y;
1200         int h= s->mb_height;
1201         uint8_t *start= buf + (size_t)(((int64_t) buf_size)*start_y/h);
1202         uint8_t *end  = buf + (size_t)(((int64_t) buf_size)*  end_y/h);
1203
1204         init_put_bits(&s->thread_context[i]->pb, start, end - start);
1205     }
1206
1207     s->picture_in_gop_number++;
1208
1209     if(load_input_picture(s, pic_arg) < 0)
1210         return -1;
1211
1212     select_input_picture(s);
1213
1214     /* output? */
1215     if(s->new_picture.data[0]){
1216         s->pict_type= s->new_picture.pict_type;
1217 //emms_c();
1218 //printf("qs:%f %f %d\n", s->new_picture.quality, s->current_picture.quality, s->qscale);
1219         MPV_frame_start(s, avctx);
1220 vbv_retry:
1221         if (encode_picture(s, s->picture_number) < 0)
1222             return -1;
1223
1224         avctx->real_pict_num  = s->picture_number;
1225         avctx->header_bits = s->header_bits;
1226         avctx->mv_bits     = s->mv_bits;
1227         avctx->misc_bits   = s->misc_bits;
1228         avctx->i_tex_bits  = s->i_tex_bits;
1229         avctx->p_tex_bits  = s->p_tex_bits;
1230         avctx->i_count     = s->i_count;
1231         avctx->p_count     = s->mb_num - s->i_count - s->skip_count; //FIXME f/b_count in avctx
1232         avctx->skip_count  = s->skip_count;
1233
1234         MPV_frame_end(s);
1235
1236         if (ENABLE_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1237             ff_mjpeg_encode_picture_trailer(s);
1238
1239         if(avctx->rc_buffer_size){
1240             RateControlContext *rcc= &s->rc_context;
1241             int max_size= rcc->buffer_index/3;
1242
1243             if(put_bits_count(&s->pb) > max_size && s->lambda < s->avctx->lmax){
1244                 s->next_lambda= FFMAX(s->lambda+1, s->lambda*(s->qscale+1) / s->qscale);
1245                 if(s->adaptive_quant){
1246                     int i;
1247                     for(i=0; i<s->mb_height*s->mb_stride; i++)
1248                         s->lambda_table[i]= FFMAX(s->lambda_table[i]+1, s->lambda_table[i]*(s->qscale+1) / s->qscale);
1249                 }
1250                 s->mb_skipped = 0;        //done in MPV_frame_start()
1251                 if(s->pict_type==P_TYPE){ //done in encode_picture() so we must undo it
1252                     if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
1253                         s->no_rounding ^= 1;
1254                 }
1255                 if(s->pict_type!=B_TYPE){
1256                     s->time_base= s->last_time_base;
1257                     s->last_non_b_time= s->time - s->pp_time;
1258                 }
1259 //                av_log(NULL, AV_LOG_ERROR, "R:%d ", s->next_lambda);
1260                 for(i=0; i<avctx->thread_count; i++){
1261                     PutBitContext *pb= &s->thread_context[i]->pb;
1262                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1263                 }
1264                 goto vbv_retry;
1265             }
1266
1267             assert(s->avctx->rc_max_rate);
1268         }
1269
1270         if(s->flags&CODEC_FLAG_PASS1)
1271             ff_write_pass1_stats(s);
1272
1273         for(i=0; i<4; i++){
1274             s->current_picture_ptr->error[i]= s->current_picture.error[i];
1275             avctx->error[i] += s->current_picture_ptr->error[i];
1276         }
1277
1278         if(s->flags&CODEC_FLAG_PASS1)
1279             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits + avctx->i_tex_bits + avctx->p_tex_bits == put_bits_count(&s->pb));
1280         flush_put_bits(&s->pb);
1281         s->frame_bits  = put_bits_count(&s->pb);
1282
1283         stuffing_count= ff_vbv_update(s, s->frame_bits);
1284         if(stuffing_count){
1285             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < stuffing_count + 50){
1286                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1287                 return -1;
1288             }
1289
1290             switch(s->codec_id){
1291             case CODEC_ID_MPEG1VIDEO:
1292             case CODEC_ID_MPEG2VIDEO:
1293                 while(stuffing_count--){
1294                     put_bits(&s->pb, 8, 0);
1295                 }
1296             break;
1297             case CODEC_ID_MPEG4:
1298                 put_bits(&s->pb, 16, 0);
1299                 put_bits(&s->pb, 16, 0x1C3);
1300                 stuffing_count -= 4;
1301                 while(stuffing_count--){
1302                     put_bits(&s->pb, 8, 0xFF);
1303                 }
1304             break;
1305             default:
1306                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1307             }
1308             flush_put_bits(&s->pb);
1309             s->frame_bits  = put_bits_count(&s->pb);
1310         }
1311
1312         /* update mpeg1/2 vbv_delay for CBR */
1313         if(s->avctx->rc_max_rate && s->avctx->rc_min_rate == s->avctx->rc_max_rate && s->out_format == FMT_MPEG1
1314            && 90000LL * (avctx->rc_buffer_size-1) <= s->avctx->rc_max_rate*0xFFFFLL){
1315             int vbv_delay;
1316
1317             assert(s->repeat_first_field==0);
1318
1319             vbv_delay= lrintf(90000 * s->rc_context.buffer_index / s->avctx->rc_max_rate);
1320             assert(vbv_delay < 0xFFFF);
1321
1322             s->vbv_delay_ptr[0] &= 0xF8;
1323             s->vbv_delay_ptr[0] |= vbv_delay>>13;
1324             s->vbv_delay_ptr[1]  = vbv_delay>>5;
1325             s->vbv_delay_ptr[2] &= 0x07;
1326             s->vbv_delay_ptr[2] |= vbv_delay<<3;
1327         }
1328         s->total_bits += s->frame_bits;
1329         avctx->frame_bits  = s->frame_bits;
1330     }else{
1331         assert((pbBufPtr(&s->pb) == s->pb.buf));
1332         s->frame_bits=0;
1333     }
1334     assert((s->frame_bits&7)==0);
1335
1336     return s->frame_bits/8;
1337 }
1338
1339 static inline void dct_single_coeff_elimination(MpegEncContext *s, int n, int threshold)
1340 {
1341     static const char tab[64]=
1342         {3,2,2,1,1,1,1,1,
1343          1,1,1,1,1,1,1,1,
1344          1,1,1,1,1,1,1,1,
1345          0,0,0,0,0,0,0,0,
1346          0,0,0,0,0,0,0,0,
1347          0,0,0,0,0,0,0,0,
1348          0,0,0,0,0,0,0,0,
1349          0,0,0,0,0,0,0,0};
1350     int score=0;
1351     int run=0;
1352     int i;
1353     DCTELEM *block= s->block[n];
1354     const int last_index= s->block_last_index[n];
1355     int skip_dc;
1356
1357     if(threshold<0){
1358         skip_dc=0;
1359         threshold= -threshold;
1360     }else
1361         skip_dc=1;
1362
1363     /* are all which we could set to zero are allready zero? */
1364     if(last_index<=skip_dc - 1) return;
1365
1366     for(i=0; i<=last_index; i++){
1367         const int j = s->intra_scantable.permutated[i];
1368         const int level = FFABS(block[j]);
1369         if(level==1){
1370             if(skip_dc && i==0) continue;
1371             score+= tab[run];
1372             run=0;
1373         }else if(level>1){
1374             return;
1375         }else{
1376             run++;
1377         }
1378     }
1379     if(score >= threshold) return;
1380     for(i=skip_dc; i<=last_index; i++){
1381         const int j = s->intra_scantable.permutated[i];
1382         block[j]=0;
1383     }
1384     if(block[0]) s->block_last_index[n]= 0;
1385     else         s->block_last_index[n]= -1;
1386 }
1387
1388 static inline void clip_coeffs(MpegEncContext *s, DCTELEM *block, int last_index)
1389 {
1390     int i;
1391     const int maxlevel= s->max_qcoeff;
1392     const int minlevel= s->min_qcoeff;
1393     int overflow=0;
1394
1395     if(s->mb_intra){
1396         i=1; //skip clipping of intra dc
1397     }else
1398         i=0;
1399
1400     for(;i<=last_index; i++){
1401         const int j= s->intra_scantable.permutated[i];
1402         int level = block[j];
1403
1404         if     (level>maxlevel){
1405             level=maxlevel;
1406             overflow++;
1407         }else if(level<minlevel){
1408             level=minlevel;
1409             overflow++;
1410         }
1411
1412         block[j]= level;
1413     }
1414
1415     if(overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1416         av_log(s->avctx, AV_LOG_INFO, "warning, clipping %d dct coefficients to %d..%d\n", overflow, minlevel, maxlevel);
1417 }
1418
1419 static void get_vissual_weight(int16_t *weight, uint8_t *ptr, int stride){
1420     int x, y;
1421 //FIXME optimize
1422     for(y=0; y<8; y++){
1423         for(x=0; x<8; x++){
1424             int x2, y2;
1425             int sum=0;
1426             int sqr=0;
1427             int count=0;
1428
1429             for(y2= FFMAX(y-1, 0); y2 < FFMIN(8, y+2); y2++){
1430                 for(x2= FFMAX(x-1, 0); x2 < FFMIN(8, x+2); x2++){
1431                     int v= ptr[x2 + y2*stride];
1432                     sum += v;
1433                     sqr += v*v;
1434                     count++;
1435                 }
1436             }
1437             weight[x + 8*y]= (36*ff_sqrt(count*sqr - sum*sum)) / count;
1438         }
1439     }
1440 }
1441
1442 static av_always_inline void encode_mb_internal(MpegEncContext *s, int motion_x, int motion_y, int mb_block_height, int mb_block_count)
1443 {
1444     int16_t weight[8][64];
1445     DCTELEM orig[8][64];
1446     const int mb_x= s->mb_x;
1447     const int mb_y= s->mb_y;
1448     int i;
1449     int skip_dct[8];
1450     int dct_offset   = s->linesize*8; //default for progressive frames
1451     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1452     int wrap_y, wrap_c;
1453
1454     for(i=0; i<mb_block_count; i++) skip_dct[i]=s->skipdct;
1455
1456     if(s->adaptive_quant){
1457         const int last_qp= s->qscale;
1458         const int mb_xy= mb_x + mb_y*s->mb_stride;
1459
1460         s->lambda= s->lambda_table[mb_xy];
1461         update_qscale(s);
1462
1463         if(!(s->flags&CODEC_FLAG_QP_RD)){
1464             s->qscale= s->current_picture_ptr->qscale_table[mb_xy];
1465             s->dquant= s->qscale - last_qp;
1466
1467             if(s->out_format==FMT_H263){
1468                 s->dquant= av_clip(s->dquant, -2, 2);
1469
1470                 if(s->codec_id==CODEC_ID_MPEG4){
1471                     if(!s->mb_intra){
1472                         if(s->pict_type == B_TYPE){
1473                             if(s->dquant&1 || s->mv_dir&MV_DIRECT)
1474                                 s->dquant= 0;
1475                         }
1476                         if(s->mv_type==MV_TYPE_8X8)
1477                             s->dquant=0;
1478                     }
1479                 }
1480             }
1481         }
1482         ff_set_qscale(s, last_qp + s->dquant);
1483     }else if(s->flags&CODEC_FLAG_QP_RD)
1484         ff_set_qscale(s, s->qscale + s->dquant);
1485
1486     wrap_y = s->linesize;
1487     wrap_c = s->uvlinesize;
1488     ptr_y = s->new_picture.data[0] + (mb_y * 16 * wrap_y) + mb_x * 16;
1489     ptr_cb = s->new_picture.data[1] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1490     ptr_cr = s->new_picture.data[2] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1491
1492     if(mb_x*16+16 > s->width || mb_y*16+16 > s->height){
1493         uint8_t *ebuf= s->edge_emu_buffer + 32;
1494         ff_emulated_edge_mc(ebuf            , ptr_y , wrap_y,16,16,mb_x*16,mb_y*16, s->width   , s->height);
1495         ptr_y= ebuf;
1496         ff_emulated_edge_mc(ebuf+18*wrap_y  , ptr_cb, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1497         ptr_cb= ebuf+18*wrap_y;
1498         ff_emulated_edge_mc(ebuf+18*wrap_y+8, ptr_cr, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1499         ptr_cr= ebuf+18*wrap_y+8;
1500     }
1501
1502     if (s->mb_intra) {
1503         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1504             int progressive_score, interlaced_score;
1505
1506             s->interlaced_dct=0;
1507             progressive_score= s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y, 8)
1508                               +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y*8, NULL, wrap_y, 8) - 400;
1509
1510             if(progressive_score > 0){
1511                 interlaced_score = s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y*2, 8)
1512                                   +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y  , NULL, wrap_y*2, 8);
1513                 if(progressive_score > interlaced_score){
1514                     s->interlaced_dct=1;
1515
1516                     dct_offset= wrap_y;
1517                     wrap_y<<=1;
1518                     if (s->chroma_format == CHROMA_422)
1519                         wrap_c<<=1;
1520                 }
1521             }
1522         }
1523
1524         s->dsp.get_pixels(s->block[0], ptr_y                 , wrap_y);
1525         s->dsp.get_pixels(s->block[1], ptr_y              + 8, wrap_y);
1526         s->dsp.get_pixels(s->block[2], ptr_y + dct_offset    , wrap_y);
1527         s->dsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
1528
1529         if(s->flags&CODEC_FLAG_GRAY){
1530             skip_dct[4]= 1;
1531             skip_dct[5]= 1;
1532         }else{
1533             s->dsp.get_pixels(s->block[4], ptr_cb, wrap_c);
1534             s->dsp.get_pixels(s->block[5], ptr_cr, wrap_c);
1535             if(!s->chroma_y_shift){ /* 422 */
1536                 s->dsp.get_pixels(s->block[6], ptr_cb + (dct_offset>>1), wrap_c);
1537                 s->dsp.get_pixels(s->block[7], ptr_cr + (dct_offset>>1), wrap_c);
1538             }
1539         }
1540     }else{
1541         op_pixels_func (*op_pix)[4];
1542         qpel_mc_func (*op_qpix)[16];
1543         uint8_t *dest_y, *dest_cb, *dest_cr;
1544
1545         dest_y  = s->dest[0];
1546         dest_cb = s->dest[1];
1547         dest_cr = s->dest[2];
1548
1549         if ((!s->no_rounding) || s->pict_type==B_TYPE){
1550             op_pix = s->dsp.put_pixels_tab;
1551             op_qpix= s->dsp.put_qpel_pixels_tab;
1552         }else{
1553             op_pix = s->dsp.put_no_rnd_pixels_tab;
1554             op_qpix= s->dsp.put_no_rnd_qpel_pixels_tab;
1555         }
1556
1557         if (s->mv_dir & MV_DIR_FORWARD) {
1558             MPV_motion(s, dest_y, dest_cb, dest_cr, 0, s->last_picture.data, op_pix, op_qpix);
1559             op_pix = s->dsp.avg_pixels_tab;
1560             op_qpix= s->dsp.avg_qpel_pixels_tab;
1561         }
1562         if (s->mv_dir & MV_DIR_BACKWARD) {
1563             MPV_motion(s, dest_y, dest_cb, dest_cr, 1, s->next_picture.data, op_pix, op_qpix);
1564         }
1565
1566         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1567             int progressive_score, interlaced_score;
1568
1569             s->interlaced_dct=0;
1570             progressive_score= s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y, 8)
1571                               +s->dsp.ildct_cmp[0](s, dest_y + wrap_y*8, ptr_y + wrap_y*8, wrap_y, 8) - 400;
1572
1573             if(s->avctx->ildct_cmp == FF_CMP_VSSE) progressive_score -= 400;
1574
1575             if(progressive_score>0){
1576                 interlaced_score = s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y*2, 8)
1577                                   +s->dsp.ildct_cmp[0](s, dest_y + wrap_y  , ptr_y + wrap_y  , wrap_y*2, 8);
1578
1579                 if(progressive_score > interlaced_score){
1580                     s->interlaced_dct=1;
1581
1582                     dct_offset= wrap_y;
1583                     wrap_y<<=1;
1584                     if (s->chroma_format == CHROMA_422)
1585                         wrap_c<<=1;
1586                 }
1587             }
1588         }
1589
1590         s->dsp.diff_pixels(s->block[0], ptr_y                 , dest_y                 , wrap_y);
1591         s->dsp.diff_pixels(s->block[1], ptr_y              + 8, dest_y              + 8, wrap_y);
1592         s->dsp.diff_pixels(s->block[2], ptr_y + dct_offset    , dest_y + dct_offset    , wrap_y);
1593         s->dsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8, dest_y + dct_offset + 8, wrap_y);
1594
1595         if(s->flags&CODEC_FLAG_GRAY){
1596             skip_dct[4]= 1;
1597             skip_dct[5]= 1;
1598         }else{
1599             s->dsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
1600             s->dsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
1601             if(!s->chroma_y_shift){ /* 422 */
1602                 s->dsp.diff_pixels(s->block[6], ptr_cb + (dct_offset>>1), dest_cb + (dct_offset>>1), wrap_c);
1603                 s->dsp.diff_pixels(s->block[7], ptr_cr + (dct_offset>>1), dest_cr + (dct_offset>>1), wrap_c);
1604             }
1605         }
1606         /* pre quantization */
1607         if(s->current_picture.mc_mb_var[s->mb_stride*mb_y+ mb_x]<2*s->qscale*s->qscale){
1608             //FIXME optimize
1609             if(s->dsp.sad[1](NULL, ptr_y               , dest_y               , wrap_y, 8) < 20*s->qscale) skip_dct[0]= 1;
1610             if(s->dsp.sad[1](NULL, ptr_y            + 8, dest_y            + 8, wrap_y, 8) < 20*s->qscale) skip_dct[1]= 1;
1611             if(s->dsp.sad[1](NULL, ptr_y +dct_offset   , dest_y +dct_offset   , wrap_y, 8) < 20*s->qscale) skip_dct[2]= 1;
1612             if(s->dsp.sad[1](NULL, ptr_y +dct_offset+ 8, dest_y +dct_offset+ 8, wrap_y, 8) < 20*s->qscale) skip_dct[3]= 1;
1613             if(s->dsp.sad[1](NULL, ptr_cb              , dest_cb              , wrap_c, 8) < 20*s->qscale) skip_dct[4]= 1;
1614             if(s->dsp.sad[1](NULL, ptr_cr              , dest_cr              , wrap_c, 8) < 20*s->qscale) skip_dct[5]= 1;
1615             if(!s->chroma_y_shift){ /* 422 */
1616                 if(s->dsp.sad[1](NULL, ptr_cb +(dct_offset>>1), dest_cb +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[6]= 1;
1617                 if(s->dsp.sad[1](NULL, ptr_cr +(dct_offset>>1), dest_cr +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[7]= 1;
1618             }
1619         }
1620     }
1621
1622     if(s->avctx->quantizer_noise_shaping){
1623         if(!skip_dct[0]) get_vissual_weight(weight[0], ptr_y                 , wrap_y);
1624         if(!skip_dct[1]) get_vissual_weight(weight[1], ptr_y              + 8, wrap_y);
1625         if(!skip_dct[2]) get_vissual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
1626         if(!skip_dct[3]) get_vissual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
1627         if(!skip_dct[4]) get_vissual_weight(weight[4], ptr_cb                , wrap_c);
1628         if(!skip_dct[5]) get_vissual_weight(weight[5], ptr_cr                , wrap_c);
1629         if(!s->chroma_y_shift){ /* 422 */
1630             if(!skip_dct[6]) get_vissual_weight(weight[6], ptr_cb + (dct_offset>>1), wrap_c);
1631             if(!skip_dct[7]) get_vissual_weight(weight[7], ptr_cr + (dct_offset>>1), wrap_c);
1632         }
1633         memcpy(orig[0], s->block[0], sizeof(DCTELEM)*64*mb_block_count);
1634     }
1635
1636     /* DCT & quantize */
1637     assert(s->out_format!=FMT_MJPEG || s->qscale==8);
1638     {
1639         for(i=0;i<mb_block_count;i++) {
1640             if(!skip_dct[i]){
1641                 int overflow;
1642                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
1643             // FIXME we could decide to change to quantizer instead of clipping
1644             // JS: I don't think that would be a good idea it could lower quality instead
1645             //     of improve it. Just INTRADC clipping deserves changes in quantizer
1646                 if (overflow) clip_coeffs(s, s->block[i], s->block_last_index[i]);
1647             }else
1648                 s->block_last_index[i]= -1;
1649         }
1650         if(s->avctx->quantizer_noise_shaping){
1651             for(i=0;i<mb_block_count;i++) {
1652                 if(!skip_dct[i]){
1653                     s->block_last_index[i] = dct_quantize_refine(s, s->block[i], weight[i], orig[i], i, s->qscale);
1654                 }
1655             }
1656         }
1657
1658         if(s->luma_elim_threshold && !s->mb_intra)
1659             for(i=0; i<4; i++)
1660                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
1661         if(s->chroma_elim_threshold && !s->mb_intra)
1662             for(i=4; i<mb_block_count; i++)
1663                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
1664
1665         if(s->flags & CODEC_FLAG_CBP_RD){
1666             for(i=0;i<mb_block_count;i++) {
1667                 if(s->block_last_index[i] == -1)
1668                     s->coded_score[i]= INT_MAX/256;
1669             }
1670         }
1671     }
1672
1673     if((s->flags&CODEC_FLAG_GRAY) && s->mb_intra){
1674         s->block_last_index[4]=
1675         s->block_last_index[5]= 0;
1676         s->block[4][0]=
1677         s->block[5][0]= (1024 + s->c_dc_scale/2)/ s->c_dc_scale;
1678     }
1679
1680     //non c quantize code returns incorrect block_last_index FIXME
1681     if(s->alternate_scan && s->dct_quantize != dct_quantize_c){
1682         for(i=0; i<mb_block_count; i++){
1683             int j;
1684             if(s->block_last_index[i]>0){
1685                 for(j=63; j>0; j--){
1686                     if(s->block[i][ s->intra_scantable.permutated[j] ]) break;
1687                 }
1688                 s->block_last_index[i]= j;
1689             }
1690         }
1691     }
1692
1693     /* huffman encode */
1694     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
1695     case CODEC_ID_MPEG1VIDEO:
1696     case CODEC_ID_MPEG2VIDEO:
1697         if (ENABLE_MPEG1VIDEO_ENCODER || ENABLE_MPEG2VIDEO_ENCODER)
1698             mpeg1_encode_mb(s, s->block, motion_x, motion_y);
1699         break;
1700     case CODEC_ID_MPEG4:
1701         mpeg4_encode_mb(s, s->block, motion_x, motion_y); break;
1702     case CODEC_ID_MSMPEG4V2:
1703     case CODEC_ID_MSMPEG4V3:
1704     case CODEC_ID_WMV1:
1705         if (ENABLE_MSMPEG4_ENCODER)
1706             msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
1707         break;
1708     case CODEC_ID_WMV2:
1709         if (ENABLE_WMV2_ENCODER)
1710             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
1711         break;
1712     case CODEC_ID_H261:
1713         if (ENABLE_H261_ENCODER)
1714             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
1715         break;
1716     case CODEC_ID_H263:
1717     case CODEC_ID_H263P:
1718     case CODEC_ID_FLV1:
1719     case CODEC_ID_RV10:
1720     case CODEC_ID_RV20:
1721         h263_encode_mb(s, s->block, motion_x, motion_y); break;
1722     case CODEC_ID_MJPEG:
1723         if (ENABLE_MJPEG_ENCODER)
1724             ff_mjpeg_encode_mb(s, s->block);
1725         break;
1726     default:
1727         assert(0);
1728     }
1729 }
1730
1731 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
1732 {
1733     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
1734     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
1735 }
1736
1737 void ff_copy_bits(PutBitContext *pb, uint8_t *src, int length)
1738 {
1739     const uint16_t *srcw= (uint16_t*)src;
1740     int words= length>>4;
1741     int bits= length&15;
1742     int i;
1743
1744     if(length==0) return;
1745
1746     if(words < 16){
1747         for(i=0; i<words; i++) put_bits(pb, 16, be2me_16(srcw[i]));
1748     }else if(put_bits_count(pb)&7){
1749         for(i=0; i<words; i++) put_bits(pb, 16, be2me_16(srcw[i]));
1750     }else{
1751         for(i=0; put_bits_count(pb)&31; i++)
1752             put_bits(pb, 8, src[i]);
1753         flush_put_bits(pb);
1754         memcpy(pbBufPtr(pb), src+i, 2*words-i);
1755         skip_put_bytes(pb, 2*words-i);
1756     }
1757
1758     put_bits(pb, bits, be2me_16(srcw[words])>>(16-bits));
1759 }
1760
1761 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
1762     int i;
1763
1764     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster then a loop?
1765
1766     /* mpeg1 */
1767     d->mb_skip_run= s->mb_skip_run;
1768     for(i=0; i<3; i++)
1769         d->last_dc[i]= s->last_dc[i];
1770
1771     /* statistics */
1772     d->mv_bits= s->mv_bits;
1773     d->i_tex_bits= s->i_tex_bits;
1774     d->p_tex_bits= s->p_tex_bits;
1775     d->i_count= s->i_count;
1776     d->f_count= s->f_count;
1777     d->b_count= s->b_count;
1778     d->skip_count= s->skip_count;
1779     d->misc_bits= s->misc_bits;
1780     d->last_bits= 0;
1781
1782     d->mb_skipped= 0;
1783     d->qscale= s->qscale;
1784     d->dquant= s->dquant;
1785 }
1786
1787 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
1788     int i;
1789
1790     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
1791     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster then a loop?
1792
1793     /* mpeg1 */
1794     d->mb_skip_run= s->mb_skip_run;
1795     for(i=0; i<3; i++)
1796         d->last_dc[i]= s->last_dc[i];
1797
1798     /* statistics */
1799     d->mv_bits= s->mv_bits;
1800     d->i_tex_bits= s->i_tex_bits;
1801     d->p_tex_bits= s->p_tex_bits;
1802     d->i_count= s->i_count;
1803     d->f_count= s->f_count;
1804     d->b_count= s->b_count;
1805     d->skip_count= s->skip_count;
1806     d->misc_bits= s->misc_bits;
1807
1808     d->mb_intra= s->mb_intra;
1809     d->mb_skipped= s->mb_skipped;
1810     d->mv_type= s->mv_type;
1811     d->mv_dir= s->mv_dir;
1812     d->pb= s->pb;
1813     if(s->data_partitioning){
1814         d->pb2= s->pb2;
1815         d->tex_pb= s->tex_pb;
1816     }
1817     d->block= s->block;
1818     for(i=0; i<8; i++)
1819         d->block_last_index[i]= s->block_last_index[i];
1820     d->interlaced_dct= s->interlaced_dct;
1821     d->qscale= s->qscale;
1822 }
1823
1824 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
1825                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
1826                            int *dmin, int *next_block, int motion_x, int motion_y)
1827 {
1828     int score;
1829     uint8_t *dest_backup[3];
1830
1831     copy_context_before_encode(s, backup, type);
1832
1833     s->block= s->blocks[*next_block];
1834     s->pb= pb[*next_block];
1835     if(s->data_partitioning){
1836         s->pb2   = pb2   [*next_block];
1837         s->tex_pb= tex_pb[*next_block];
1838     }
1839
1840     if(*next_block){
1841         memcpy(dest_backup, s->dest, sizeof(s->dest));
1842         s->dest[0] = s->rd_scratchpad;
1843         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
1844         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
1845         assert(s->linesize >= 32); //FIXME
1846     }
1847
1848     encode_mb(s, motion_x, motion_y);
1849
1850     score= put_bits_count(&s->pb);
1851     if(s->data_partitioning){
1852         score+= put_bits_count(&s->pb2);
1853         score+= put_bits_count(&s->tex_pb);
1854     }
1855
1856     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
1857         MPV_decode_mb(s, s->block);
1858
1859         score *= s->lambda2;
1860         score += sse_mb(s) << FF_LAMBDA_SHIFT;
1861     }
1862
1863     if(*next_block){
1864         memcpy(s->dest, dest_backup, sizeof(s->dest));
1865     }
1866
1867     if(score<*dmin){
1868         *dmin= score;
1869         *next_block^=1;
1870
1871         copy_context_after_encode(best, s, type);
1872     }
1873 }
1874
1875 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
1876     uint32_t *sq = ff_squareTbl + 256;
1877     int acc=0;
1878     int x,y;
1879
1880     if(w==16 && h==16)
1881         return s->dsp.sse[0](NULL, src1, src2, stride, 16);
1882     else if(w==8 && h==8)
1883         return s->dsp.sse[1](NULL, src1, src2, stride, 8);
1884
1885     for(y=0; y<h; y++){
1886         for(x=0; x<w; x++){
1887             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
1888         }
1889     }
1890
1891     assert(acc>=0);
1892
1893     return acc;
1894 }
1895
1896 static int sse_mb(MpegEncContext *s){
1897     int w= 16;
1898     int h= 16;
1899
1900     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
1901     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
1902
1903     if(w==16 && h==16)
1904       if(s->avctx->mb_cmp == FF_CMP_NSSE){
1905         return  s->dsp.nsse[0](s, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
1906                +s->dsp.nsse[1](s, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
1907                +s->dsp.nsse[1](s, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
1908       }else{
1909         return  s->dsp.sse[0](NULL, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
1910                +s->dsp.sse[1](NULL, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
1911                +s->dsp.sse[1](NULL, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
1912       }
1913     else
1914         return  sse(s, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
1915                +sse(s, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
1916                +sse(s, s->new_picture.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
1917 }
1918
1919 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
1920     MpegEncContext *s= arg;
1921
1922
1923     s->me.pre_pass=1;
1924     s->me.dia_size= s->avctx->pre_dia_size;
1925     s->first_slice_line=1;
1926     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
1927         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
1928             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
1929         }
1930         s->first_slice_line=0;
1931     }
1932
1933     s->me.pre_pass=0;
1934
1935     return 0;
1936 }
1937
1938 static int estimate_motion_thread(AVCodecContext *c, void *arg){
1939     MpegEncContext *s= arg;
1940
1941     ff_check_alignment();
1942
1943     s->me.dia_size= s->avctx->dia_size;
1944     s->first_slice_line=1;
1945     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
1946         s->mb_x=0; //for block init below
1947         ff_init_block_index(s);
1948         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
1949             s->block_index[0]+=2;
1950             s->block_index[1]+=2;
1951             s->block_index[2]+=2;
1952             s->block_index[3]+=2;
1953
1954             /* compute motion vector & mb_type and store in context */
1955             if(s->pict_type==B_TYPE)
1956                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
1957             else
1958                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
1959         }
1960         s->first_slice_line=0;
1961     }
1962     return 0;
1963 }
1964
1965 static int mb_var_thread(AVCodecContext *c, void *arg){
1966     MpegEncContext *s= arg;
1967     int mb_x, mb_y;
1968
1969     ff_check_alignment();
1970
1971     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
1972         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
1973             int xx = mb_x * 16;
1974             int yy = mb_y * 16;
1975             uint8_t *pix = s->new_picture.data[0] + (yy * s->linesize) + xx;
1976             int varc;
1977             int sum = s->dsp.pix_sum(pix, s->linesize);
1978
1979             varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)(sum*sum))>>8) + 500 + 128)>>8;
1980
1981             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
1982             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
1983             s->me.mb_var_sum_temp    += varc;
1984         }
1985     }
1986     return 0;
1987 }
1988
1989 static void write_slice_end(MpegEncContext *s){
1990     if(s->codec_id==CODEC_ID_MPEG4){
1991         if(s->partitioned_frame){
1992             ff_mpeg4_merge_partitions(s);
1993         }
1994
1995         ff_mpeg4_stuffing(&s->pb);
1996     }else if(ENABLE_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
1997         ff_mjpeg_encode_stuffing(&s->pb);
1998     }
1999
2000     align_put_bits(&s->pb);
2001     flush_put_bits(&s->pb);
2002
2003     if((s->flags&CODEC_FLAG_PASS1) && !s->partitioned_frame)
2004         s->misc_bits+= get_bits_diff(s);
2005 }
2006
2007 static int encode_thread(AVCodecContext *c, void *arg){
2008     MpegEncContext *s= arg;
2009     int mb_x, mb_y, pdif = 0;
2010     int i, j;
2011     MpegEncContext best_s, backup_s;
2012     uint8_t bit_buf[2][MAX_MB_BYTES];
2013     uint8_t bit_buf2[2][MAX_MB_BYTES];
2014     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2015     PutBitContext pb[2], pb2[2], tex_pb[2];
2016 //printf("%d->%d\n", s->resync_mb_y, s->end_mb_y);
2017
2018     ff_check_alignment();
2019
2020     for(i=0; i<2; i++){
2021         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2022         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2023         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2024     }
2025
2026     s->last_bits= put_bits_count(&s->pb);
2027     s->mv_bits=0;
2028     s->misc_bits=0;
2029     s->i_tex_bits=0;
2030     s->p_tex_bits=0;
2031     s->i_count=0;
2032     s->f_count=0;
2033     s->b_count=0;
2034     s->skip_count=0;
2035
2036     for(i=0; i<3; i++){
2037         /* init last dc values */
2038         /* note: quant matrix value (8) is implied here */
2039         s->last_dc[i] = 128 << s->intra_dc_precision;
2040
2041         s->current_picture.error[i] = 0;
2042     }
2043     s->mb_skip_run = 0;
2044     memset(s->last_mv, 0, sizeof(s->last_mv));
2045
2046     s->last_mv_dir = 0;
2047
2048     switch(s->codec_id){
2049     case CODEC_ID_H263:
2050     case CODEC_ID_H263P:
2051     case CODEC_ID_FLV1:
2052         s->gob_index = ff_h263_get_gob_height(s);
2053         break;
2054     case CODEC_ID_MPEG4:
2055         if(s->partitioned_frame)
2056             ff_mpeg4_init_partitions(s);
2057         break;
2058     }
2059
2060     s->resync_mb_x=0;
2061     s->resync_mb_y=0;
2062     s->first_slice_line = 1;
2063     s->ptr_lastgob = s->pb.buf;
2064     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2065 //    printf("row %d at %X\n", s->mb_y, (int)s);
2066         s->mb_x=0;
2067         s->mb_y= mb_y;
2068
2069         ff_set_qscale(s, s->qscale);
2070         ff_init_block_index(s);
2071
2072         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2073             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2074             int mb_type= s->mb_type[xy];
2075 //            int d;
2076             int dmin= INT_MAX;
2077             int dir;
2078
2079             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2080                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2081                 return -1;
2082             }
2083             if(s->data_partitioning){
2084                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2085                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2086                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2087                     return -1;
2088                 }
2089             }
2090
2091             s->mb_x = mb_x;
2092             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2093             ff_update_block_index(s);
2094
2095             if(ENABLE_H261_ENCODER && s->codec_id == CODEC_ID_H261){
2096                 ff_h261_reorder_mb_index(s);
2097                 xy= s->mb_y*s->mb_stride + s->mb_x;
2098                 mb_type= s->mb_type[xy];
2099             }
2100
2101             /* write gob / video packet header  */
2102             if(s->rtp_mode){
2103                 int current_packet_size, is_gob_start;
2104
2105                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2106
2107                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2108
2109                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2110
2111                 switch(s->codec_id){
2112                 case CODEC_ID_H263:
2113                 case CODEC_ID_H263P:
2114                     if(!s->h263_slice_structured)
2115                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2116                     break;
2117                 case CODEC_ID_MPEG2VIDEO:
2118                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2119                 case CODEC_ID_MPEG1VIDEO:
2120                     if(s->mb_skip_run) is_gob_start=0;
2121                     break;
2122                 }
2123
2124                 if(is_gob_start){
2125                     if(s->start_mb_y != mb_y || mb_x!=0){
2126                         write_slice_end(s);
2127
2128                         if(s->codec_id==CODEC_ID_MPEG4 && s->partitioned_frame){
2129                             ff_mpeg4_init_partitions(s);
2130                         }
2131                     }
2132
2133                     assert((put_bits_count(&s->pb)&7) == 0);
2134                     current_packet_size= pbBufPtr(&s->pb) - s->ptr_lastgob;
2135
2136                     if(s->avctx->error_rate && s->resync_mb_x + s->resync_mb_y > 0){
2137                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2138                         int d= 100 / s->avctx->error_rate;
2139                         if(r % d == 0){
2140                             current_packet_size=0;
2141 #ifndef ALT_BITSTREAM_WRITER
2142                             s->pb.buf_ptr= s->ptr_lastgob;
2143 #endif
2144                             assert(pbBufPtr(&s->pb) == s->ptr_lastgob);
2145                         }
2146                     }
2147
2148                     if (s->avctx->rtp_callback){
2149                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2150                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2151                     }
2152
2153                     switch(s->codec_id){
2154                     case CODEC_ID_MPEG4:
2155                         ff_mpeg4_encode_video_packet_header(s);
2156                         ff_mpeg4_clean_buffers(s);
2157                     break;
2158                     case CODEC_ID_MPEG1VIDEO:
2159                     case CODEC_ID_MPEG2VIDEO:
2160                         if (ENABLE_MPEG1VIDEO_ENCODER || ENABLE_MPEG2VIDEO_ENCODER) {
2161                             ff_mpeg1_encode_slice_header(s);
2162                             ff_mpeg1_clean_buffers(s);
2163                         }
2164                     break;
2165                     case CODEC_ID_H263:
2166                     case CODEC_ID_H263P:
2167                         h263_encode_gob_header(s, mb_y);
2168                     break;
2169                     }
2170
2171                     if(s->flags&CODEC_FLAG_PASS1){
2172                         int bits= put_bits_count(&s->pb);
2173                         s->misc_bits+= bits - s->last_bits;
2174                         s->last_bits= bits;
2175                     }
2176
2177                     s->ptr_lastgob += current_packet_size;
2178                     s->first_slice_line=1;
2179                     s->resync_mb_x=mb_x;
2180                     s->resync_mb_y=mb_y;
2181                 }
2182             }
2183
2184             if(  (s->resync_mb_x   == s->mb_x)
2185                && s->resync_mb_y+1 == s->mb_y){
2186                 s->first_slice_line=0;
2187             }
2188
2189             s->mb_skipped=0;
2190             s->dquant=0; //only for QP_RD
2191
2192             if(mb_type & (mb_type-1) || (s->flags & CODEC_FLAG_QP_RD)){ // more than 1 MB type possible or CODEC_FLAG_QP_RD
2193                 int next_block=0;
2194                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2195
2196                 copy_context_before_encode(&backup_s, s, -1);
2197                 backup_s.pb= s->pb;
2198                 best_s.data_partitioning= s->data_partitioning;
2199                 best_s.partitioned_frame= s->partitioned_frame;
2200                 if(s->data_partitioning){
2201                     backup_s.pb2= s->pb2;
2202                     backup_s.tex_pb= s->tex_pb;
2203                 }
2204
2205                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2206                     s->mv_dir = MV_DIR_FORWARD;
2207                     s->mv_type = MV_TYPE_16X16;
2208                     s->mb_intra= 0;
2209                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2210                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2211                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2212                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2213                 }
2214                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2215                     s->mv_dir = MV_DIR_FORWARD;
2216                     s->mv_type = MV_TYPE_FIELD;
2217                     s->mb_intra= 0;
2218                     for(i=0; i<2; i++){
2219                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2220                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2221                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2222                     }
2223                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2224                                  &dmin, &next_block, 0, 0);
2225                 }
2226                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2227                     s->mv_dir = MV_DIR_FORWARD;
2228                     s->mv_type = MV_TYPE_16X16;
2229                     s->mb_intra= 0;
2230                     s->mv[0][0][0] = 0;
2231                     s->mv[0][0][1] = 0;
2232                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2233                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2234                 }
2235                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2236                     s->mv_dir = MV_DIR_FORWARD;
2237                     s->mv_type = MV_TYPE_8X8;
2238                     s->mb_intra= 0;
2239                     for(i=0; i<4; i++){
2240                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2241                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2242                     }
2243                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2244                                  &dmin, &next_block, 0, 0);
2245                 }
2246                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2247                     s->mv_dir = MV_DIR_FORWARD;
2248                     s->mv_type = MV_TYPE_16X16;
2249                     s->mb_intra= 0;
2250                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2251                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2252                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2253                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2254                 }
2255                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2256                     s->mv_dir = MV_DIR_BACKWARD;
2257                     s->mv_type = MV_TYPE_16X16;
2258                     s->mb_intra= 0;
2259                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2260                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2261                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2262                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2263                 }
2264                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2265                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2266                     s->mv_type = MV_TYPE_16X16;
2267                     s->mb_intra= 0;
2268                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2269                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2270                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2271                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2272                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2273                                  &dmin, &next_block, 0, 0);
2274                 }
2275                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2276                     s->mv_dir = MV_DIR_FORWARD;
2277                     s->mv_type = MV_TYPE_FIELD;
2278                     s->mb_intra= 0;
2279                     for(i=0; i<2; i++){
2280                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2281                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2282                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2283                     }
2284                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2285                                  &dmin, &next_block, 0, 0);
2286                 }
2287                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2288                     s->mv_dir = MV_DIR_BACKWARD;
2289                     s->mv_type = MV_TYPE_FIELD;
2290                     s->mb_intra= 0;
2291                     for(i=0; i<2; i++){
2292                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2293                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2294                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2295                     }
2296                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2297                                  &dmin, &next_block, 0, 0);
2298                 }
2299                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2300                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2301                     s->mv_type = MV_TYPE_FIELD;
2302                     s->mb_intra= 0;
2303                     for(dir=0; dir<2; dir++){
2304                         for(i=0; i<2; i++){
2305                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2306                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2307                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2308                         }
2309                     }
2310                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2311                                  &dmin, &next_block, 0, 0);
2312                 }
2313                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2314                     s->mv_dir = 0;
2315                     s->mv_type = MV_TYPE_16X16;
2316                     s->mb_intra= 1;
2317                     s->mv[0][0][0] = 0;
2318                     s->mv[0][0][1] = 0;
2319                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2320                                  &dmin, &next_block, 0, 0);
2321                     if(s->h263_pred || s->h263_aic){
2322                         if(best_s.mb_intra)
2323                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2324                         else
2325                             ff_clean_intra_table_entries(s); //old mode?
2326                     }
2327                 }
2328
2329                 if((s->flags & CODEC_FLAG_QP_RD) && dmin < INT_MAX){
2330                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2331                         const int last_qp= backup_s.qscale;
2332                         int qpi, qp, dc[6];
2333                         DCTELEM ac[6][16];
2334                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2335                         static const int dquant_tab[4]={-1,1,-2,2};
2336
2337                         assert(backup_s.dquant == 0);
2338
2339                         //FIXME intra
2340                         s->mv_dir= best_s.mv_dir;
2341                         s->mv_type = MV_TYPE_16X16;
2342                         s->mb_intra= best_s.mb_intra;
2343                         s->mv[0][0][0] = best_s.mv[0][0][0];
2344                         s->mv[0][0][1] = best_s.mv[0][0][1];
2345                         s->mv[1][0][0] = best_s.mv[1][0][0];
2346                         s->mv[1][0][1] = best_s.mv[1][0][1];
2347
2348                         qpi = s->pict_type == B_TYPE ? 2 : 0;
2349                         for(; qpi<4; qpi++){
2350                             int dquant= dquant_tab[qpi];
2351                             qp= last_qp + dquant;
2352                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2353                                 continue;
2354                             backup_s.dquant= dquant;
2355                             if(s->mb_intra && s->dc_val[0]){
2356                                 for(i=0; i<6; i++){
2357                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2358                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(DCTELEM)*16);
2359                                 }
2360                             }
2361
2362                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2363                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2364                             if(best_s.qscale != qp){
2365                                 if(s->mb_intra && s->dc_val[0]){
2366                                     for(i=0; i<6; i++){
2367                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2368                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(DCTELEM)*16);
2369                                     }
2370                                 }
2371                             }
2372                         }
2373                     }
2374                 }
2375                 if(mb_type&CANDIDATE_MB_TYPE_DIRECT){
2376                     int mx= s->b_direct_mv_table[xy][0];
2377                     int my= s->b_direct_mv_table[xy][1];
2378
2379                     backup_s.dquant = 0;
2380                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2381                     s->mb_intra= 0;
2382                     ff_mpeg4_set_direct_mv(s, mx, my);
2383                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2384                                  &dmin, &next_block, mx, my);
2385                 }
2386                 if(mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2387                     backup_s.dquant = 0;
2388                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2389                     s->mb_intra= 0;
2390                     ff_mpeg4_set_direct_mv(s, 0, 0);
2391                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2392                                  &dmin, &next_block, 0, 0);
2393                 }
2394                 if(!best_s.mb_intra && s->flags2&CODEC_FLAG2_SKIP_RD){
2395                     int coded=0;
2396                     for(i=0; i<6; i++)
2397                         coded |= s->block_last_index[i];
2398                     if(coded){
2399                         int mx,my;
2400                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2401                         if(best_s.mv_dir & MV_DIRECT){
2402                             mx=my=0; //FIXME find the one we actually used
2403                             ff_mpeg4_set_direct_mv(s, mx, my);
2404                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2405                             mx= s->mv[1][0][0];
2406                             my= s->mv[1][0][1];
2407                         }else{
2408                             mx= s->mv[0][0][0];
2409                             my= s->mv[0][0][1];
2410                         }
2411
2412                         s->mv_dir= best_s.mv_dir;
2413                         s->mv_type = best_s.mv_type;
2414                         s->mb_intra= 0;
2415 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2416                         s->mv[0][0][1] = best_s.mv[0][0][1];
2417                         s->mv[1][0][0] = best_s.mv[1][0][0];
2418                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2419                         backup_s.dquant= 0;
2420                         s->skipdct=1;
2421                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2422                                         &dmin, &next_block, mx, my);
2423                         s->skipdct=0;
2424                     }
2425                 }
2426
2427                 s->current_picture.qscale_table[xy]= best_s.qscale;
2428
2429                 copy_context_after_encode(s, &best_s, -1);
2430
2431                 pb_bits_count= put_bits_count(&s->pb);
2432                 flush_put_bits(&s->pb);
2433                 ff_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2434                 s->pb= backup_s.pb;
2435
2436                 if(s->data_partitioning){
2437                     pb2_bits_count= put_bits_count(&s->pb2);
2438                     flush_put_bits(&s->pb2);
2439                     ff_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2440                     s->pb2= backup_s.pb2;
2441
2442                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2443                     flush_put_bits(&s->tex_pb);
2444                     ff_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2445                     s->tex_pb= backup_s.tex_pb;
2446                 }
2447                 s->last_bits= put_bits_count(&s->pb);
2448
2449                 if (s->out_format == FMT_H263 && s->pict_type!=B_TYPE)
2450                     ff_h263_update_motion_val(s);
2451
2452                 if(next_block==0){ //FIXME 16 vs linesize16
2453                     s->dsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
2454                     s->dsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2455                     s->dsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2456                 }
2457
2458                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
2459                     MPV_decode_mb(s, s->block);
2460             } else {
2461                 int motion_x = 0, motion_y = 0;
2462                 s->mv_type=MV_TYPE_16X16;
2463                 // only one MB-Type possible
2464
2465                 switch(mb_type){
2466                 case CANDIDATE_MB_TYPE_INTRA:
2467                     s->mv_dir = 0;
2468                     s->mb_intra= 1;
2469                     motion_x= s->mv[0][0][0] = 0;
2470                     motion_y= s->mv[0][0][1] = 0;
2471                     break;
2472                 case CANDIDATE_MB_TYPE_INTER:
2473                     s->mv_dir = MV_DIR_FORWARD;
2474                     s->mb_intra= 0;
2475                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
2476                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
2477                     break;
2478                 case CANDIDATE_MB_TYPE_INTER_I:
2479                     s->mv_dir = MV_DIR_FORWARD;
2480                     s->mv_type = MV_TYPE_FIELD;
2481                     s->mb_intra= 0;
2482                     for(i=0; i<2; i++){
2483                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2484                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2485                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2486                     }
2487                     break;
2488                 case CANDIDATE_MB_TYPE_INTER4V:
2489                     s->mv_dir = MV_DIR_FORWARD;
2490                     s->mv_type = MV_TYPE_8X8;
2491                     s->mb_intra= 0;
2492                     for(i=0; i<4; i++){
2493                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2494                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2495                     }
2496                     break;
2497                 case CANDIDATE_MB_TYPE_DIRECT:
2498                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2499                     s->mb_intra= 0;
2500                     motion_x=s->b_direct_mv_table[xy][0];
2501                     motion_y=s->b_direct_mv_table[xy][1];
2502                     ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
2503                     break;
2504                 case CANDIDATE_MB_TYPE_DIRECT0:
2505                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2506                     s->mb_intra= 0;
2507                     ff_mpeg4_set_direct_mv(s, 0, 0);
2508                     break;
2509                 case CANDIDATE_MB_TYPE_BIDIR:
2510                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2511                     s->mb_intra= 0;
2512                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2513                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2514                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2515                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2516                     break;
2517                 case CANDIDATE_MB_TYPE_BACKWARD:
2518                     s->mv_dir = MV_DIR_BACKWARD;
2519                     s->mb_intra= 0;
2520                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2521                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2522                     break;
2523                 case CANDIDATE_MB_TYPE_FORWARD:
2524                     s->mv_dir = MV_DIR_FORWARD;
2525                     s->mb_intra= 0;
2526                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2527                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2528 //                    printf(" %d %d ", motion_x, motion_y);
2529                     break;
2530                 case CANDIDATE_MB_TYPE_FORWARD_I:
2531                     s->mv_dir = MV_DIR_FORWARD;
2532                     s->mv_type = MV_TYPE_FIELD;
2533                     s->mb_intra= 0;
2534                     for(i=0; i<2; i++){
2535                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2536                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2537                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2538                     }
2539                     break;
2540                 case CANDIDATE_MB_TYPE_BACKWARD_I:
2541                     s->mv_dir = MV_DIR_BACKWARD;
2542                     s->mv_type = MV_TYPE_FIELD;
2543                     s->mb_intra= 0;
2544                     for(i=0; i<2; i++){
2545                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2546                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2547                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2548                     }
2549                     break;
2550                 case CANDIDATE_MB_TYPE_BIDIR_I:
2551                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2552                     s->mv_type = MV_TYPE_FIELD;
2553                     s->mb_intra= 0;
2554                     for(dir=0; dir<2; dir++){
2555                         for(i=0; i<2; i++){
2556                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2557                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2558                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2559                         }
2560                     }
2561                     break;
2562                 default:
2563                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
2564                 }
2565
2566                 encode_mb(s, motion_x, motion_y);
2567
2568                 // RAL: Update last macroblock type
2569                 s->last_mv_dir = s->mv_dir;
2570
2571                 if (s->out_format == FMT_H263 && s->pict_type!=B_TYPE)
2572                     ff_h263_update_motion_val(s);
2573
2574                 MPV_decode_mb(s, s->block);
2575             }
2576
2577             /* clean the MV table in IPS frames for direct mode in B frames */
2578             if(s->mb_intra /* && I,P,S_TYPE */){
2579                 s->p_mv_table[xy][0]=0;
2580                 s->p_mv_table[xy][1]=0;
2581             }
2582
2583             if(s->flags&CODEC_FLAG_PSNR){
2584                 int w= 16;
2585                 int h= 16;
2586
2587                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2588                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2589
2590                 s->current_picture.error[0] += sse(
2591                     s, s->new_picture.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
2592                     s->dest[0], w, h, s->linesize);
2593                 s->current_picture.error[1] += sse(
2594                     s, s->new_picture.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,
2595                     s->dest[1], w>>1, h>>1, s->uvlinesize);
2596                 s->current_picture.error[2] += sse(
2597                     s, s->new_picture    .data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,
2598                     s->dest[2], w>>1, h>>1, s->uvlinesize);
2599             }
2600             if(s->loop_filter){
2601                 if(s->out_format == FMT_H263)
2602                     ff_h263_loop_filter(s);
2603             }
2604 //printf("MB %d %d bits\n", s->mb_x+s->mb_y*s->mb_stride, put_bits_count(&s->pb));
2605         }
2606     }
2607
2608     //not beautiful here but we must write it before flushing so it has to be here
2609     if (ENABLE_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == I_TYPE)
2610         msmpeg4_encode_ext_header(s);
2611
2612     write_slice_end(s);
2613
2614     /* Send the last GOB if RTP */
2615     if (s->avctx->rtp_callback) {
2616         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
2617         pdif = pbBufPtr(&s->pb) - s->ptr_lastgob;
2618         /* Call the RTP callback to send the last GOB */
2619         emms_c();
2620         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
2621     }
2622
2623     return 0;
2624 }
2625
2626 #define MERGE(field) dst->field += src->field; src->field=0
2627 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
2628     MERGE(me.scene_change_score);
2629     MERGE(me.mc_mb_var_sum_temp);
2630     MERGE(me.mb_var_sum_temp);
2631 }
2632
2633 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
2634     int i;
2635
2636     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
2637     MERGE(dct_count[1]);
2638     MERGE(mv_bits);
2639     MERGE(i_tex_bits);
2640     MERGE(p_tex_bits);
2641     MERGE(i_count);
2642     MERGE(f_count);
2643     MERGE(b_count);
2644     MERGE(skip_count);
2645     MERGE(misc_bits);
2646     MERGE(error_count);
2647     MERGE(padding_bug_score);
2648     MERGE(current_picture.error[0]);
2649     MERGE(current_picture.error[1]);
2650     MERGE(current_picture.error[2]);
2651
2652     if(dst->avctx->noise_reduction){
2653         for(i=0; i<64; i++){
2654             MERGE(dct_error_sum[0][i]);
2655             MERGE(dct_error_sum[1][i]);
2656         }
2657     }
2658
2659     assert(put_bits_count(&src->pb) % 8 ==0);
2660     assert(put_bits_count(&dst->pb) % 8 ==0);
2661     ff_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
2662     flush_put_bits(&dst->pb);
2663 }
2664
2665 static int estimate_qp(MpegEncContext *s, int dry_run){
2666     if (s->next_lambda){
2667         s->current_picture_ptr->quality=
2668         s->current_picture.quality = s->next_lambda;
2669         if(!dry_run) s->next_lambda= 0;
2670     } else if (!s->fixed_qscale) {
2671         s->current_picture_ptr->quality=
2672         s->current_picture.quality = ff_rate_estimate_qscale(s, dry_run);
2673         if (s->current_picture.quality < 0)
2674             return -1;
2675     }
2676
2677     if(s->adaptive_quant){
2678         switch(s->codec_id){
2679         case CODEC_ID_MPEG4:
2680             ff_clean_mpeg4_qscales(s);
2681             break;
2682         case CODEC_ID_H263:
2683         case CODEC_ID_H263P:
2684         case CODEC_ID_FLV1:
2685             ff_clean_h263_qscales(s);
2686             break;
2687         }
2688
2689         s->lambda= s->lambda_table[0];
2690         //FIXME broken
2691     }else
2692         s->lambda= s->current_picture.quality;
2693 //printf("%d %d\n", s->avctx->global_quality, s->current_picture.quality);
2694     update_qscale(s);
2695     return 0;
2696 }
2697
2698 static int encode_picture(MpegEncContext *s, int picture_number)
2699 {
2700     int i;
2701     int bits;
2702
2703     s->picture_number = picture_number;
2704
2705     /* Reset the average MB variance */
2706     s->me.mb_var_sum_temp    =
2707     s->me.mc_mb_var_sum_temp = 0;
2708
2709     /* we need to initialize some time vars before we can encode b-frames */
2710     // RAL: Condition added for MPEG1VIDEO
2711     if (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->h263_msmpeg4))
2712         ff_set_mpeg4_time(s, s->picture_number);  //FIXME rename and use has_b_frames or similar
2713
2714     s->me.scene_change_score=0;
2715
2716 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME ratedistoration
2717
2718     if(s->pict_type==I_TYPE){
2719         if(s->msmpeg4_version >= 3) s->no_rounding=1;
2720         else                        s->no_rounding=0;
2721     }else if(s->pict_type!=B_TYPE){
2722         if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
2723             s->no_rounding ^= 1;
2724     }
2725
2726     if(s->flags & CODEC_FLAG_PASS2){
2727         if (estimate_qp(s,1) < 0)
2728             return -1;
2729         ff_get_2pass_fcode(s);
2730     }else if(!(s->flags & CODEC_FLAG_QSCALE)){
2731         if(s->pict_type==B_TYPE)
2732             s->lambda= s->last_lambda_for[s->pict_type];
2733         else
2734             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
2735         update_qscale(s);
2736     }
2737
2738     s->mb_intra=0; //for the rate distortion & bit compare functions
2739     for(i=1; i<s->avctx->thread_count; i++){
2740         ff_update_duplicate_context(s->thread_context[i], s);
2741     }
2742
2743     ff_init_me(s);
2744
2745     /* Estimate motion for every MB */
2746     if(s->pict_type != I_TYPE){
2747         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
2748         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
2749         if(s->pict_type != B_TYPE && s->avctx->me_threshold==0){
2750             if((s->avctx->pre_me && s->last_non_b_pict_type==I_TYPE) || s->avctx->pre_me==2){
2751                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, (void**)&(s->thread_context[0]), NULL, s->avctx->thread_count);
2752             }
2753         }
2754
2755         s->avctx->execute(s->avctx, estimate_motion_thread, (void**)&(s->thread_context[0]), NULL, s->avctx->thread_count);
2756     }else /* if(s->pict_type == I_TYPE) */{
2757         /* I-Frame */
2758         for(i=0; i<s->mb_stride*s->mb_height; i++)
2759             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
2760
2761         if(!s->fixed_qscale){
2762             /* finding spatial complexity for I-frame rate control */
2763             s->avctx->execute(s->avctx, mb_var_thread, (void**)&(s->thread_context[0]), NULL, s->avctx->thread_count);
2764         }
2765     }
2766     for(i=1; i<s->avctx->thread_count; i++){
2767         merge_context_after_me(s, s->thread_context[i]);
2768     }
2769     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
2770     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
2771     emms_c();
2772
2773     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == P_TYPE){
2774         s->pict_type= I_TYPE;
2775         for(i=0; i<s->mb_stride*s->mb_height; i++)
2776             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
2777 //printf("Scene change detected, encoding as I Frame %d %d\n", s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
2778     }
2779
2780     if(!s->umvplus){
2781         if(s->pict_type==P_TYPE || s->pict_type==S_TYPE) {
2782             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
2783
2784             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2785                 int a,b;
2786                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
2787                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
2788                 s->f_code= FFMAX(s->f_code, FFMAX(a,b));
2789             }
2790
2791             ff_fix_long_p_mvs(s);
2792             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
2793             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2794                 int j;
2795                 for(i=0; i<2; i++){
2796                     for(j=0; j<2; j++)
2797                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
2798                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
2799                 }
2800             }
2801         }
2802
2803         if(s->pict_type==B_TYPE){
2804             int a, b;
2805
2806             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
2807             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
2808             s->f_code = FFMAX(a, b);
2809
2810             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
2811             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
2812             s->b_code = FFMAX(a, b);
2813
2814             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
2815             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
2816             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
2817             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
2818             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2819                 int dir, j;
2820                 for(dir=0; dir<2; dir++){
2821                     for(i=0; i<2; i++){
2822                         for(j=0; j<2; j++){
2823                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
2824                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
2825                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
2826                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
2827                         }
2828                     }
2829                 }
2830             }
2831         }
2832     }
2833
2834     if (estimate_qp(s, 0) < 0)
2835         return -1;
2836
2837     if(s->qscale < 3 && s->max_qcoeff<=128 && s->pict_type==I_TYPE && !(s->flags & CODEC_FLAG_QSCALE))
2838         s->qscale= 3; //reduce clipping problems
2839
2840     if (s->out_format == FMT_MJPEG) {
2841         /* for mjpeg, we do include qscale in the matrix */
2842         s->intra_matrix[0] = ff_mpeg1_default_intra_matrix[0];
2843         for(i=1;i<64;i++){
2844             int j= s->dsp.idct_permutation[i];
2845
2846             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
2847         }
2848         convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
2849                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
2850         s->qscale= 8;
2851     }
2852
2853     //FIXME var duplication
2854     s->current_picture_ptr->key_frame=
2855     s->current_picture.key_frame= s->pict_type == I_TYPE; //FIXME pic_ptr
2856     s->current_picture_ptr->pict_type=
2857     s->current_picture.pict_type= s->pict_type;
2858
2859     if(s->current_picture.key_frame)
2860         s->picture_in_gop_number=0;
2861
2862     s->last_bits= put_bits_count(&s->pb);
2863     switch(s->out_format) {
2864     case FMT_MJPEG:
2865         if (ENABLE_MJPEG_ENCODER)
2866             ff_mjpeg_encode_picture_header(s);
2867         break;
2868     case FMT_H261:
2869         if (ENABLE_H261_ENCODER)
2870             ff_h261_encode_picture_header(s, picture_number);
2871         break;
2872     case FMT_H263:
2873         if (ENABLE_WMV2_ENCODER && s->codec_id == CODEC_ID_WMV2)
2874             ff_wmv2_encode_picture_header(s, picture_number);
2875         else if (ENABLE_MSMPEG4_ENCODER && s->h263_msmpeg4)
2876             msmpeg4_encode_picture_header(s, picture_number);
2877         else if (s->h263_pred)
2878             mpeg4_encode_picture_header(s, picture_number);
2879         else if (ENABLE_RV10_ENCODER && s->codec_id == CODEC_ID_RV10)
2880             rv10_encode_picture_header(s, picture_number);
2881         else if (ENABLE_RV20_ENCODER && s->codec_id == CODEC_ID_RV20)
2882             rv20_encode_picture_header(s, picture_number);
2883         else if (s->codec_id == CODEC_ID_FLV1)
2884             ff_flv_encode_picture_header(s, picture_number);
2885         else
2886             h263_encode_picture_header(s, picture_number);
2887         break;
2888     case FMT_MPEG1:
2889         if (ENABLE_MPEG1VIDEO_ENCODER || ENABLE_MPEG2VIDEO_ENCODER)
2890             mpeg1_encode_picture_header(s, picture_number);
2891         break;
2892     case FMT_H264:
2893         break;
2894     default:
2895         assert(0);
2896     }
2897     bits= put_bits_count(&s->pb);
2898     s->header_bits= bits - s->last_bits;
2899
2900     for(i=1; i<s->avctx->thread_count; i++){
2901         update_duplicate_context_after_me(s->thread_context[i], s);
2902     }
2903     s->avctx->execute(s->avctx, encode_thread, (void**)&(s->thread_context[0]), NULL, s->avctx->thread_count);
2904     for(i=1; i<s->avctx->thread_count; i++){
2905         merge_context_after_encode(s, s->thread_context[i]);
2906     }
2907     emms_c();
2908     return 0;
2909 }
2910
2911 void  denoise_dct_c(MpegEncContext *s, DCTELEM *block){
2912     const int intra= s->mb_intra;
2913     int i;
2914
2915     s->dct_count[intra]++;
2916
2917     for(i=0; i<64; i++){
2918         int level= block[i];
2919
2920         if(level){
2921             if(level>0){
2922                 s->dct_error_sum[intra][i] += level;
2923                 level -= s->dct_offset[intra][i];
2924                 if(level<0) level=0;
2925             }else{
2926                 s->dct_error_sum[intra][i] -= level;
2927                 level += s->dct_offset[intra][i];
2928                 if(level>0) level=0;
2929             }
2930             block[i]= level;
2931         }
2932     }
2933 }
2934
2935 int dct_quantize_trellis_c(MpegEncContext *s,
2936                         DCTELEM *block, int n,
2937                         int qscale, int *overflow){
2938     const int *qmat;
2939     const uint8_t *scantable= s->intra_scantable.scantable;
2940     const uint8_t *perm_scantable= s->intra_scantable.permutated;
2941     int max=0;
2942     unsigned int threshold1, threshold2;
2943     int bias=0;
2944     int run_tab[65];
2945     int level_tab[65];
2946     int score_tab[65];
2947     int survivor[65];
2948     int survivor_count;
2949     int last_run=0;
2950     int last_level=0;
2951     int last_score= 0;
2952     int last_i;
2953     int coeff[2][64];
2954     int coeff_count[64];
2955     int qmul, qadd, start_i, last_non_zero, i, dc;
2956     const int esc_length= s->ac_esc_length;
2957     uint8_t * length;
2958     uint8_t * last_length;
2959     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
2960
2961     s->dsp.fdct (block);
2962
2963     if(s->dct_error_sum)
2964         s->denoise_dct(s, block);
2965     qmul= qscale*16;
2966     qadd= ((qscale-1)|1)*8;
2967
2968     if (s->mb_intra) {
2969         int q;
2970         if (!s->h263_aic) {
2971             if (n < 4)
2972                 q = s->y_dc_scale;
2973             else
2974                 q = s->c_dc_scale;
2975             q = q << 3;
2976         } else{
2977             /* For AIC we skip quant/dequant of INTRADC */
2978             q = 1 << 3;
2979             qadd=0;
2980         }
2981
2982         /* note: block[0] is assumed to be positive */
2983         block[0] = (block[0] + (q >> 1)) / q;
2984         start_i = 1;
2985         last_non_zero = 0;
2986         qmat = s->q_intra_matrix[qscale];
2987         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
2988             bias= 1<<(QMAT_SHIFT-1);
2989         length     = s->intra_ac_vlc_length;
2990         last_length= s->intra_ac_vlc_last_length;
2991     } else {
2992         start_i = 0;
2993         last_non_zero = -1;
2994         qmat = s->q_inter_matrix[qscale];
2995         length     = s->inter_ac_vlc_length;
2996         last_length= s->inter_ac_vlc_last_length;
2997     }
2998     last_i= start_i;
2999
3000     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3001     threshold2= (threshold1<<1);
3002
3003     for(i=63; i>=start_i; i--) {
3004         const int j = scantable[i];
3005         int level = block[j] * qmat[j];
3006
3007         if(((unsigned)(level+threshold1))>threshold2){
3008             last_non_zero = i;
3009             break;
3010         }
3011     }
3012
3013     for(i=start_i; i<=last_non_zero; i++) {
3014         const int j = scantable[i];
3015         int level = block[j] * qmat[j];
3016
3017 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3018 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3019         if(((unsigned)(level+threshold1))>threshold2){
3020             if(level>0){
3021                 level= (bias + level)>>QMAT_SHIFT;
3022                 coeff[0][i]= level;
3023                 coeff[1][i]= level-1;
3024 //                coeff[2][k]= level-2;
3025             }else{
3026                 level= (bias - level)>>QMAT_SHIFT;
3027                 coeff[0][i]= -level;
3028                 coeff[1][i]= -level+1;
3029 //                coeff[2][k]= -level+2;
3030             }
3031             coeff_count[i]= FFMIN(level, 2);
3032             assert(coeff_count[i]);
3033             max |=level;
3034         }else{
3035             coeff[0][i]= (level>>31)|1;
3036             coeff_count[i]= 1;
3037         }
3038     }
3039
3040     *overflow= s->max_qcoeff < max; //overflow might have happened
3041
3042     if(last_non_zero < start_i){
3043         memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3044         return last_non_zero;
3045     }
3046
3047     score_tab[start_i]= 0;
3048     survivor[0]= start_i;
3049     survivor_count= 1;
3050
3051     for(i=start_i; i<=last_non_zero; i++){
3052         int level_index, j;
3053         const int dct_coeff= FFABS(block[ scantable[i] ]);
3054         const int zero_distoration= dct_coeff*dct_coeff;
3055         int best_score=256*256*256*120;
3056         for(level_index=0; level_index < coeff_count[i]; level_index++){
3057             int distoration;
3058             int level= coeff[level_index][i];
3059             const int alevel= FFABS(level);
3060             int unquant_coeff;
3061
3062             assert(level);
3063
3064             if(s->out_format == FMT_H263){
3065                 unquant_coeff= alevel*qmul + qadd;
3066             }else{ //MPEG1
3067                 j= s->dsp.idct_permutation[ scantable[i] ]; //FIXME optimize
3068                 if(s->mb_intra){
3069                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3070                         unquant_coeff =   (unquant_coeff - 1) | 1;
3071                 }else{
3072                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3073                         unquant_coeff =   (unquant_coeff - 1) | 1;
3074                 }
3075                 unquant_coeff<<= 3;
3076             }
3077
3078             distoration= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distoration;
3079             level+=64;
3080             if((level&(~127)) == 0){
3081                 for(j=survivor_count-1; j>=0; j--){
3082                     int run= i - survivor[j];
3083                     int score= distoration + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3084                     score += score_tab[i-run];
3085
3086                     if(score < best_score){
3087                         best_score= score;
3088                         run_tab[i+1]= run;
3089                         level_tab[i+1]= level-64;
3090                     }
3091                 }
3092
3093                 if(s->out_format == FMT_H263){
3094                     for(j=survivor_count-1; j>=0; j--){
3095                         int run= i - survivor[j];
3096                         int score= distoration + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3097                         score += score_tab[i-run];
3098                         if(score < last_score){
3099                             last_score= score;
3100                             last_run= run;
3101                             last_level= level-64;
3102                             last_i= i+1;
3103                         }
3104                     }
3105                 }
3106             }else{
3107                 distoration += esc_length*lambda;
3108                 for(j=survivor_count-1; j>=0; j--){
3109                     int run= i - survivor[j];
3110                     int score= distoration + score_tab[i-run];
3111
3112                     if(score < best_score){
3113                         best_score= score;
3114                         run_tab[i+1]= run;
3115                         level_tab[i+1]= level-64;
3116                     }
3117                 }
3118
3119                 if(s->out_format == FMT_H263){
3120                   for(j=survivor_count-1; j>=0; j--){
3121                         int run= i - survivor[j];
3122                         int score= distoration + score_tab[i-run];
3123                         if(score < last_score){
3124                             last_score= score;
3125                             last_run= run;
3126                             last_level= level-64;
3127                             last_i= i+1;
3128                         }
3129                     }
3130                 }
3131             }
3132         }
3133
3134         score_tab[i+1]= best_score;
3135
3136         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
3137         if(last_non_zero <= 27){
3138             for(; survivor_count; survivor_count--){
3139                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3140                     break;
3141             }
3142         }else{
3143             for(; survivor_count; survivor_count--){
3144                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3145                     break;
3146             }
3147         }
3148
3149         survivor[ survivor_count++ ]= i+1;
3150     }
3151
3152     if(s->out_format != FMT_H263){
3153         last_score= 256*256*256*120;
3154         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3155             int score= score_tab[i];
3156             if(i) score += lambda*2; //FIXME exacter?
3157
3158             if(score < last_score){
3159                 last_score= score;
3160                 last_i= i;
3161                 last_level= level_tab[i];
3162                 last_run= run_tab[i];
3163             }
3164         }
3165     }
3166
3167     s->coded_score[n] = last_score;
3168
3169     dc= FFABS(block[0]);
3170     last_non_zero= last_i - 1;
3171     memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3172
3173     if(last_non_zero < start_i)
3174         return last_non_zero;
3175
3176     if(last_non_zero == 0 && start_i == 0){
3177         int best_level= 0;
3178         int best_score= dc * dc;
3179
3180         for(i=0; i<coeff_count[0]; i++){
3181             int level= coeff[i][0];
3182             int alevel= FFABS(level);
3183             int unquant_coeff, score, distortion;
3184
3185             if(s->out_format == FMT_H263){
3186                     unquant_coeff= (alevel*qmul + qadd)>>3;
3187             }else{ //MPEG1
3188                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3189                     unquant_coeff =   (unquant_coeff - 1) | 1;
3190             }
3191             unquant_coeff = (unquant_coeff + 4) >> 3;
3192             unquant_coeff<<= 3 + 3;
3193
3194             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3195             level+=64;
3196             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3197             else                    score= distortion + esc_length*lambda;
3198
3199             if(score < best_score){
3200                 best_score= score;
3201                 best_level= level - 64;
3202             }
3203         }
3204         block[0]= best_level;
3205         s->coded_score[n] = best_score - dc*dc;
3206         if(best_level == 0) return -1;
3207         else                return last_non_zero;
3208     }
3209
3210     i= last_i;
3211     assert(last_level);
3212
3213     block[ perm_scantable[last_non_zero] ]= last_level;
3214     i -= last_run + 1;
3215
3216     for(; i>start_i; i -= run_tab[i] + 1){
3217         block[ perm_scantable[i-1] ]= level_tab[i];
3218     }
3219
3220     return last_non_zero;
3221 }
3222
3223 //#define REFINE_STATS 1
3224 static int16_t basis[64][64];
3225
3226 static void build_basis(uint8_t *perm){
3227     int i, j, x, y;
3228     emms_c();
3229     for(i=0; i<8; i++){
3230         for(j=0; j<8; j++){
3231             for(y=0; y<8; y++){
3232                 for(x=0; x<8; x++){
3233                     double s= 0.25*(1<<BASIS_SHIFT);
3234                     int index= 8*i + j;
3235                     int perm_index= perm[index];
3236                     if(i==0) s*= sqrt(0.5);
3237                     if(j==0) s*= sqrt(0.5);
3238                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3239                 }
3240             }
3241         }
3242     }
3243 }
3244
3245 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3246                         DCTELEM *block, int16_t *weight, DCTELEM *orig,
3247                         int n, int qscale){
3248     int16_t rem[64];
3249     DECLARE_ALIGNED_16(DCTELEM, d1[64]);
3250     const int *qmat;
3251     const uint8_t *scantable= s->intra_scantable.scantable;
3252     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3253 //    unsigned int threshold1, threshold2;
3254 //    int bias=0;
3255     int run_tab[65];
3256     int prev_run=0;
3257     int prev_level=0;
3258     int qmul, qadd, start_i, last_non_zero, i, dc;
3259     uint8_t * length;
3260     uint8_t * last_length;
3261     int lambda;
3262     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3263 #ifdef REFINE_STATS
3264 static int count=0;
3265 static int after_last=0;
3266 static int to_zero=0;
3267 static int from_zero=0;
3268 static int raise=0;
3269 static int lower=0;
3270 static int messed_sign=0;
3271 #endif
3272
3273     if(basis[0][0] == 0)
3274         build_basis(s->dsp.idct_permutation);
3275
3276     qmul= qscale*2;
3277     qadd= (qscale-1)|1;
3278     if (s->mb_intra) {
3279         if (!s->h263_aic) {
3280             if (n < 4)
3281                 q = s->y_dc_scale;
3282             else
3283                 q = s->c_dc_scale;
3284         } else{
3285             /* For AIC we skip quant/dequant of INTRADC */
3286             q = 1;
3287             qadd=0;
3288         }
3289         q <<= RECON_SHIFT-3;
3290         /* note: block[0] is assumed to be positive */
3291         dc= block[0]*q;
3292 //        block[0] = (block[0] + (q >> 1)) / q;
3293         start_i = 1;
3294         qmat = s->q_intra_matrix[qscale];
3295 //        if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3296 //            bias= 1<<(QMAT_SHIFT-1);
3297         length     = s->intra_ac_vlc_length;
3298         last_length= s->intra_ac_vlc_last_length;
3299     } else {
3300         dc= 0;
3301         start_i = 0;
3302         qmat = s->q_inter_matrix[qscale];
3303         length     = s->inter_ac_vlc_length;
3304         last_length= s->inter_ac_vlc_last_length;
3305     }
3306     last_non_zero = s->block_last_index[n];
3307
3308 #ifdef REFINE_STATS
3309 {START_TIMER
3310 #endif
3311     dc += (1<<(RECON_SHIFT-1));
3312     for(i=0; i<64; i++){
3313         rem[i]= dc - (orig[i]<<RECON_SHIFT); //FIXME  use orig dirrectly instead of copying to rem[]
3314     }
3315 #ifdef REFINE_STATS
3316 STOP_TIMER("memset rem[]")}
3317 #endif
3318     sum=0;
3319     for(i=0; i<64; i++){
3320         int one= 36;
3321         int qns=4;
3322         int w;
3323
3324         w= FFABS(weight[i]) + qns*one;
3325         w= 15 + (48*qns*one + w/2)/w; // 16 .. 63
3326
3327         weight[i] = w;
3328 //        w=weight[i] = (63*qns + (w/2)) / w;
3329
3330         assert(w>0);
3331         assert(w<(1<<6));
3332         sum += w*w;
3333     }
3334     lambda= sum*(uint64_t)s->lambda2 >> (FF_LAMBDA_SHIFT - 6 + 6 + 6 + 6);
3335 #ifdef REFINE_STATS
3336 {START_TIMER
3337 #endif
3338     run=0;
3339     rle_index=0;
3340     for(i=start_i; i<=last_non_zero; i++){
3341         int j= perm_scantable[i];
3342         const int level= block[j];
3343         int coeff;
3344
3345         if(level){
3346             if(level<0) coeff= qmul*level - qadd;
3347             else        coeff= qmul*level + qadd;
3348             run_tab[rle_index++]=run;
3349             run=0;
3350
3351             s->dsp.add_8x8basis(rem, basis[j], coeff);
3352         }else{
3353             run++;
3354         }
3355     }
3356 #ifdef REFINE_STATS
3357 if(last_non_zero>0){
3358 STOP_TIMER("init rem[]")
3359 }
3360 }
3361
3362 {START_TIMER
3363 #endif
3364     for(;;){
3365         int best_score=s->dsp.try_8x8basis(rem, weight, basis[0], 0);
3366         int best_coeff=0;
3367         int best_change=0;
3368         int run2, best_unquant_change=0, analyze_gradient;
3369 #ifdef REFINE_STATS
3370 {START_TIMER
3371 #endif
3372         analyze_gradient = last_non_zero > 2 || s->avctx->quantizer_noise_shaping >= 3;
3373
3374         if(analyze_gradient){
3375 #ifdef REFINE_STATS
3376 {START_TIMER
3377 #endif
3378             for(i=0; i<64; i++){
3379                 int w= weight[i];
3380
3381                 d1[i] = (rem[i]*w*w + (1<<(RECON_SHIFT+12-1)))>>(RECON_SHIFT+12);
3382             }
3383 #ifdef REFINE_STATS
3384 STOP_TIMER("rem*w*w")}
3385 {START_TIMER
3386 #endif
3387             s->dsp.fdct(d1);
3388 #ifdef REFINE_STATS
3389 STOP_TIMER("dct")}
3390 #endif