mpeg12enc: add intra_vlc private option.
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of Libav.
9  *
10  * Libav is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * Libav is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with Libav; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include "libavutil/intmath.h"
31 #include "libavutil/mathematics.h"
32 #include "avcodec.h"
33 #include "dsputil.h"
34 #include "mpegvideo.h"
35 #include "mpegvideo_common.h"
36 #include "h263.h"
37 #include "mjpegenc.h"
38 #include "msmpeg4.h"
39 #include "faandct.h"
40 #include "thread.h"
41 #include "aandcttab.h"
42 #include "flv.h"
43 #include "mpeg4video.h"
44 #include "internal.h"
45 #include <limits.h>
46
47 //#undef NDEBUG
48 //#include <assert.h>
49
50 static int encode_picture(MpegEncContext *s, int picture_number);
51 static int dct_quantize_refine(MpegEncContext *s, DCTELEM *block, int16_t *weight, DCTELEM *orig, int n, int qscale);
52 static int sse_mb(MpegEncContext *s);
53 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block);
54 static int dct_quantize_trellis_c(MpegEncContext *s, DCTELEM *block, int n, int qscale, int *overflow);
55
56 /* enable all paranoid tests for rounding, overflows, etc... */
57 //#define PARANOID
58
59 //#define DEBUG
60
61 static uint8_t default_mv_penalty[MAX_FCODE+1][MAX_MV*2+1];
62 static uint8_t default_fcode_tab[MAX_MV*2+1];
63
64 void ff_convert_matrix(DSPContext *dsp, int (*qmat)[64], uint16_t (*qmat16)[2][64],
65                            const uint16_t *quant_matrix, int bias, int qmin, int qmax, int intra)
66 {
67     int qscale;
68     int shift=0;
69
70     for(qscale=qmin; qscale<=qmax; qscale++){
71         int i;
72         if (dsp->fdct == ff_jpeg_fdct_islow_8 ||
73             dsp->fdct == ff_jpeg_fdct_islow_10
74 #ifdef FAAN_POSTSCALE
75             || dsp->fdct == ff_faandct
76 #endif
77             ) {
78             for(i=0;i<64;i++) {
79                 const int j= dsp->idct_permutation[i];
80                 /* 16 <= qscale * quant_matrix[i] <= 7905 */
81                 /* 19952             <= ff_aanscales[i] * qscale * quant_matrix[i]               <= 249205026 */
82                 /* (1 << 36) / 19952 >= (1 << 36) / (ff_aanscales[i] * qscale * quant_matrix[i]) >= (1 << 36) / 249205026 */
83                 /* 3444240           >= (1 << 36) / (ff_aanscales[i] * qscale * quant_matrix[i]) >= 275 */
84
85                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
86                                 (qscale * quant_matrix[j]));
87             }
88         } else if (dsp->fdct == fdct_ifast
89 #ifndef FAAN_POSTSCALE
90                    || dsp->fdct == ff_faandct
91 #endif
92                    ) {
93             for(i=0;i<64;i++) {
94                 const int j= dsp->idct_permutation[i];
95                 /* 16 <= qscale * quant_matrix[i] <= 7905 */
96                 /* 19952             <= ff_aanscales[i] * qscale * quant_matrix[i]               <= 249205026 */
97                 /* (1 << 36) / 19952 >= (1 << 36) / (ff_aanscales[i] * qscale * quant_matrix[i]) >= (1<<36)/249205026 */
98                 /* 3444240           >= (1 << 36) / (ff_aanscales[i] * qscale * quant_matrix[i]) >= 275 */
99
100                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) /
101                                 (ff_aanscales[i] * qscale * quant_matrix[j]));
102             }
103         } else {
104             for(i=0;i<64;i++) {
105                 const int j= dsp->idct_permutation[i];
106                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
107                    So 16           <= qscale * quant_matrix[i]             <= 7905
108                    so (1<<19) / 16 >= (1<<19) / (qscale * quant_matrix[i]) >= (1<<19) / 7905
109                    so 32768        >= (1<<19) / (qscale * quant_matrix[i]) >= 67
110                 */
111                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / (qscale * quant_matrix[j]));
112 //                qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) / (qscale * quant_matrix[i]);
113                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / (qscale * quant_matrix[j]);
114
115                 if(qmat16[qscale][0][i]==0 || qmat16[qscale][0][i]==128*256) qmat16[qscale][0][i]=128*256-1;
116                 qmat16[qscale][1][i]= ROUNDED_DIV(bias<<(16-QUANT_BIAS_SHIFT), qmat16[qscale][0][i]);
117             }
118         }
119
120         for(i=intra; i<64; i++){
121             int64_t max= 8191;
122             if (dsp->fdct == fdct_ifast
123 #ifndef FAAN_POSTSCALE
124                    || dsp->fdct == ff_faandct
125 #endif
126                    ) {
127                 max = (8191LL*ff_aanscales[i]) >> 14;
128             }
129             while(((max * qmat[qscale][i]) >> shift) > INT_MAX){
130                 shift++;
131             }
132         }
133     }
134     if(shift){
135         av_log(NULL, AV_LOG_INFO, "Warning, QMAT_SHIFT is larger than %d, overflows possible\n", QMAT_SHIFT - shift);
136     }
137 }
138
139 static inline void update_qscale(MpegEncContext *s){
140     s->qscale= (s->lambda*139 + FF_LAMBDA_SCALE*64) >> (FF_LAMBDA_SHIFT + 7);
141     s->qscale= av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
142
143     s->lambda2= (s->lambda*s->lambda + FF_LAMBDA_SCALE/2) >> FF_LAMBDA_SHIFT;
144 }
145
146 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix){
147     int i;
148
149     if(matrix){
150         put_bits(pb, 1, 1);
151         for(i=0;i<64;i++) {
152             put_bits(pb, 8, matrix[ ff_zigzag_direct[i] ]);
153         }
154     }else
155         put_bits(pb, 1, 0);
156 }
157
158 /**
159  * init s->current_picture.qscale_table from s->lambda_table
160  */
161 void ff_init_qscale_tab(MpegEncContext *s){
162     int8_t * const qscale_table = s->current_picture.f.qscale_table;
163     int i;
164
165     for(i=0; i<s->mb_num; i++){
166         unsigned int lam= s->lambda_table[ s->mb_index2xy[i] ];
167         int qp= (lam*139 + FF_LAMBDA_SCALE*64) >> (FF_LAMBDA_SHIFT + 7);
168         qscale_table[ s->mb_index2xy[i] ]= av_clip(qp, s->avctx->qmin, s->avctx->qmax);
169     }
170 }
171
172 static void copy_picture_attributes(MpegEncContext *s, AVFrame *dst, AVFrame *src){
173     int i;
174
175     dst->pict_type              = src->pict_type;
176     dst->quality                = src->quality;
177     dst->coded_picture_number   = src->coded_picture_number;
178     dst->display_picture_number = src->display_picture_number;
179 //    dst->reference              = src->reference;
180     dst->pts                    = src->pts;
181     dst->interlaced_frame       = src->interlaced_frame;
182     dst->top_field_first        = src->top_field_first;
183
184     if(s->avctx->me_threshold){
185         if(!src->motion_val[0])
186             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_val not set!\n");
187         if(!src->mb_type)
188             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.mb_type not set!\n");
189         if(!src->ref_index[0])
190             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.ref_index not set!\n");
191         if(src->motion_subsample_log2 != dst->motion_subsample_log2)
192             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_subsample_log2 doesn't match! (%d!=%d)\n",
193             src->motion_subsample_log2, dst->motion_subsample_log2);
194
195         memcpy(dst->mb_type, src->mb_type, s->mb_stride * s->mb_height * sizeof(dst->mb_type[0]));
196
197         for(i=0; i<2; i++){
198             int stride= ((16*s->mb_width )>>src->motion_subsample_log2) + 1;
199             int height= ((16*s->mb_height)>>src->motion_subsample_log2);
200
201             if(src->motion_val[i] && src->motion_val[i] != dst->motion_val[i]){
202                 memcpy(dst->motion_val[i], src->motion_val[i], 2*stride*height*sizeof(int16_t));
203             }
204             if(src->ref_index[i] && src->ref_index[i] != dst->ref_index[i]){
205                 memcpy(dst->ref_index[i], src->ref_index[i], s->mb_stride*4*s->mb_height*sizeof(int8_t));
206             }
207         }
208     }
209 }
210
211 static void update_duplicate_context_after_me(MpegEncContext *dst, MpegEncContext *src){
212 #define COPY(a) dst->a= src->a
213     COPY(pict_type);
214     COPY(current_picture);
215     COPY(f_code);
216     COPY(b_code);
217     COPY(qscale);
218     COPY(lambda);
219     COPY(lambda2);
220     COPY(picture_in_gop_number);
221     COPY(gop_picture_number);
222     COPY(frame_pred_frame_dct); //FIXME don't set in encode_header
223     COPY(progressive_frame); //FIXME don't set in encode_header
224     COPY(partitioned_frame); //FIXME don't set in encode_header
225 #undef COPY
226 }
227
228 /**
229  * sets the given MpegEncContext to defaults for encoding.
230  * the changed fields will not depend upon the prior state of the MpegEncContext.
231  */
232 static void MPV_encode_defaults(MpegEncContext *s){
233     int i;
234     MPV_common_defaults(s);
235
236     for(i=-16; i<16; i++){
237         default_fcode_tab[i + MAX_MV]= 1;
238     }
239     s->me.mv_penalty= default_mv_penalty;
240     s->fcode_tab= default_fcode_tab;
241 }
242
243 /* init video encoder */
244 av_cold int MPV_encode_init(AVCodecContext *avctx)
245 {
246     MpegEncContext *s = avctx->priv_data;
247     int i;
248     int chroma_h_shift, chroma_v_shift;
249
250     MPV_encode_defaults(s);
251
252     switch (avctx->codec_id) {
253     case CODEC_ID_MPEG2VIDEO:
254         if(avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P){
255             av_log(avctx, AV_LOG_ERROR, "only YUV420 and YUV422 are supported\n");
256             return -1;
257         }
258         break;
259     case CODEC_ID_LJPEG:
260         if(avctx->pix_fmt != PIX_FMT_YUVJ420P && avctx->pix_fmt != PIX_FMT_YUVJ422P && avctx->pix_fmt != PIX_FMT_YUVJ444P && avctx->pix_fmt != PIX_FMT_BGRA &&
261            ((avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P && avctx->pix_fmt != PIX_FMT_YUV444P) || avctx->strict_std_compliance>FF_COMPLIANCE_UNOFFICIAL)){
262             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in LJPEG\n");
263             return -1;
264         }
265         break;
266     case CODEC_ID_MJPEG:
267         if(avctx->pix_fmt != PIX_FMT_YUVJ420P && avctx->pix_fmt != PIX_FMT_YUVJ422P &&
268            ((avctx->pix_fmt != PIX_FMT_YUV420P && avctx->pix_fmt != PIX_FMT_YUV422P) || avctx->strict_std_compliance>FF_COMPLIANCE_UNOFFICIAL)){
269             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
270             return -1;
271         }
272         break;
273     default:
274         if(avctx->pix_fmt != PIX_FMT_YUV420P){
275             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
276             return -1;
277         }
278     }
279
280     switch (avctx->pix_fmt) {
281     case PIX_FMT_YUVJ422P:
282     case PIX_FMT_YUV422P:
283         s->chroma_format = CHROMA_422;
284         break;
285     case PIX_FMT_YUVJ420P:
286     case PIX_FMT_YUV420P:
287     default:
288         s->chroma_format = CHROMA_420;
289         break;
290     }
291
292     s->bit_rate = avctx->bit_rate;
293     s->width = avctx->width;
294     s->height = avctx->height;
295     if(avctx->gop_size > 600 && avctx->strict_std_compliance>FF_COMPLIANCE_EXPERIMENTAL){
296         av_log(avctx, AV_LOG_ERROR, "Warning keyframe interval too large! reducing it ...\n");
297         avctx->gop_size=600;
298     }
299     s->gop_size = avctx->gop_size;
300     s->avctx = avctx;
301     s->flags= avctx->flags;
302     s->flags2= avctx->flags2;
303     s->max_b_frames= avctx->max_b_frames;
304     s->codec_id= avctx->codec->id;
305     s->luma_elim_threshold  = avctx->luma_elim_threshold;
306     s->chroma_elim_threshold= avctx->chroma_elim_threshold;
307     s->strict_std_compliance= avctx->strict_std_compliance;
308     s->data_partitioning= avctx->flags & CODEC_FLAG_PART;
309     s->quarter_sample= (avctx->flags & CODEC_FLAG_QPEL)!=0;
310     s->mpeg_quant= avctx->mpeg_quant;
311     s->rtp_mode= !!avctx->rtp_payload_size;
312     s->intra_dc_precision= avctx->intra_dc_precision;
313     s->user_specified_pts = AV_NOPTS_VALUE;
314
315     if (s->gop_size <= 1) {
316         s->intra_only = 1;
317         s->gop_size = 12;
318     } else {
319         s->intra_only = 0;
320     }
321
322     s->me_method = avctx->me_method;
323
324     /* Fixed QSCALE */
325     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
326
327     s->adaptive_quant= (   s->avctx->lumi_masking
328                         || s->avctx->dark_masking
329                         || s->avctx->temporal_cplx_masking
330                         || s->avctx->spatial_cplx_masking
331                         || s->avctx->p_masking
332                         || s->avctx->border_masking
333                         || (s->flags&CODEC_FLAG_QP_RD))
334                        && !s->fixed_qscale;
335
336     s->obmc= !!(s->flags & CODEC_FLAG_OBMC);
337     s->loop_filter= !!(s->flags & CODEC_FLAG_LOOP_FILTER);
338     s->alternate_scan= !!(s->flags & CODEC_FLAG_ALT_SCAN);
339 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
340     s->intra_vlc_format= !!(s->flags2 & CODEC_FLAG2_INTRA_VLC);
341 #endif
342     s->q_scale_type= !!(s->flags2 & CODEC_FLAG2_NON_LINEAR_QUANT);
343
344     if(avctx->rc_max_rate && !avctx->rc_buffer_size){
345         av_log(avctx, AV_LOG_ERROR, "a vbv buffer size is needed, for encoding with a maximum bitrate\n");
346         return -1;
347     }
348
349     if(avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate){
350         av_log(avctx, AV_LOG_INFO, "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
351     }
352
353     if(avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate){
354         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
355         return -1;
356     }
357
358     if(avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate){
359         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
360         return -1;
361     }
362
363     if(avctx->rc_max_rate && avctx->rc_max_rate == avctx->bit_rate && avctx->rc_max_rate != avctx->rc_min_rate){
364         av_log(avctx, AV_LOG_INFO, "impossible bitrate constraints, this will fail\n");
365     }
366
367     if(avctx->rc_buffer_size && avctx->bit_rate*(int64_t)avctx->time_base.num > avctx->rc_buffer_size * (int64_t)avctx->time_base.den){
368         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
369         return -1;
370     }
371
372     if(!s->fixed_qscale && avctx->bit_rate*av_q2d(avctx->time_base) > avctx->bit_rate_tolerance){
373         av_log(avctx, AV_LOG_ERROR, "bitrate tolerance too small for bitrate\n");
374         return -1;
375     }
376
377     if(   s->avctx->rc_max_rate && s->avctx->rc_min_rate == s->avctx->rc_max_rate
378        && (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO)
379        && 90000LL * (avctx->rc_buffer_size-1) > s->avctx->rc_max_rate*0xFFFFLL){
380
381         av_log(avctx, AV_LOG_INFO, "Warning vbv_delay will be set to 0xFFFF (=VBR) as the specified vbv buffer is too large for the given bitrate!\n");
382     }
383
384     if((s->flags & CODEC_FLAG_4MV) && s->codec_id != CODEC_ID_MPEG4
385        && s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P && s->codec_id != CODEC_ID_FLV1){
386         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
387         return -1;
388     }
389
390     if(s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE){
391         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with simple mb decision\n");
392         return -1;
393     }
394
395     if(s->obmc && s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P){
396         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with H263(+)\n");
397         return -1;
398     }
399
400     if(s->quarter_sample && s->codec_id != CODEC_ID_MPEG4){
401         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
402         return -1;
403     }
404
405     if(s->data_partitioning && s->codec_id != CODEC_ID_MPEG4){
406         av_log(avctx, AV_LOG_ERROR, "data partitioning not supported by codec\n");
407         return -1;
408     }
409
410     if(s->max_b_frames && s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG1VIDEO && s->codec_id != CODEC_ID_MPEG2VIDEO){
411         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
412         return -1;
413     }
414
415     if ((s->codec_id == CODEC_ID_MPEG4 || s->codec_id == CODEC_ID_H263 ||
416          s->codec_id == CODEC_ID_H263P) &&
417         (avctx->sample_aspect_ratio.num > 255 || avctx->sample_aspect_ratio.den > 255)) {
418         av_log(avctx, AV_LOG_ERROR, "Invalid pixel aspect ratio %i/%i, limit is 255/255\n",
419                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
420         return -1;
421     }
422
423     if((s->flags & (CODEC_FLAG_INTERLACED_DCT|CODEC_FLAG_INTERLACED_ME|CODEC_FLAG_ALT_SCAN))
424        && s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG2VIDEO){
425         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
426         return -1;
427     }
428
429     if(s->mpeg_quant && s->codec_id != CODEC_ID_MPEG4){ //FIXME mpeg2 uses that too
430         av_log(avctx, AV_LOG_ERROR, "mpeg2 style quantization not supported by codec\n");
431         return -1;
432     }
433
434     if((s->flags & CODEC_FLAG_CBP_RD) && !avctx->trellis){
435         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
436         return -1;
437     }
438
439     if((s->flags & CODEC_FLAG_QP_RD) && s->avctx->mb_decision != FF_MB_DECISION_RD){
440         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
441         return -1;
442     }
443
444     if(s->avctx->scenechange_threshold < 1000000000 && (s->flags & CODEC_FLAG_CLOSED_GOP)){
445         av_log(avctx, AV_LOG_ERROR, "closed gop with scene change detection are not supported yet, set threshold to 1000000000\n");
446         return -1;
447     }
448
449     if((s->flags2 & CODEC_FLAG2_INTRA_VLC) && s->codec_id != CODEC_ID_MPEG2VIDEO){
450         av_log(avctx, AV_LOG_ERROR, "intra vlc table not supported by codec\n");
451         return -1;
452     }
453
454     if(s->flags & CODEC_FLAG_LOW_DELAY){
455         if (s->codec_id != CODEC_ID_MPEG2VIDEO){
456             av_log(avctx, AV_LOG_ERROR, "low delay forcing is only available for mpeg2\n");
457             return -1;
458         }
459         if (s->max_b_frames != 0){
460             av_log(avctx, AV_LOG_ERROR, "b frames cannot be used with low delay\n");
461             return -1;
462         }
463     }
464
465     if(s->q_scale_type == 1){
466         if(s->codec_id != CODEC_ID_MPEG2VIDEO){
467             av_log(avctx, AV_LOG_ERROR, "non linear quant is only available for mpeg2\n");
468             return -1;
469         }
470         if(avctx->qmax > 12){
471             av_log(avctx, AV_LOG_ERROR, "non linear quant only supports qmax <= 12 currently\n");
472             return -1;
473         }
474     }
475
476     if(s->avctx->thread_count > 1 && s->codec_id != CODEC_ID_MPEG4
477        && s->codec_id != CODEC_ID_MPEG1VIDEO && s->codec_id != CODEC_ID_MPEG2VIDEO
478        && (s->codec_id != CODEC_ID_H263P || !(s->flags & CODEC_FLAG_H263P_SLICE_STRUCT))){
479         av_log(avctx, AV_LOG_ERROR, "multi threaded encoding not supported by codec\n");
480         return -1;
481     }
482
483     if(s->avctx->thread_count < 1){
484         av_log(avctx, AV_LOG_ERROR, "automatic thread number detection not supported by codec, patch welcome\n");
485         return -1;
486     }
487
488     if(s->avctx->thread_count > 1)
489         s->rtp_mode= 1;
490
491     if(!avctx->time_base.den || !avctx->time_base.num){
492         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
493         return -1;
494     }
495
496     i= (INT_MAX/2+128)>>8;
497     if(avctx->me_threshold >= i){
498         av_log(avctx, AV_LOG_ERROR, "me_threshold too large, max is %d\n", i - 1);
499         return -1;
500     }
501     if(avctx->mb_threshold >= i){
502         av_log(avctx, AV_LOG_ERROR, "mb_threshold too large, max is %d\n", i - 1);
503         return -1;
504     }
505
506     if(avctx->b_frame_strategy && (avctx->flags&CODEC_FLAG_PASS2)){
507         av_log(avctx, AV_LOG_INFO, "notice: b_frame_strategy only affects the first pass\n");
508         avctx->b_frame_strategy = 0;
509     }
510
511     i= av_gcd(avctx->time_base.den, avctx->time_base.num);
512     if(i > 1){
513         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
514         avctx->time_base.den /= i;
515         avctx->time_base.num /= i;
516 //        return -1;
517     }
518
519     if(s->mpeg_quant || s->codec_id==CODEC_ID_MPEG1VIDEO || s->codec_id==CODEC_ID_MPEG2VIDEO || s->codec_id==CODEC_ID_MJPEG){
520         s->intra_quant_bias= 3<<(QUANT_BIAS_SHIFT-3); //(a + x*3/8)/x
521         s->inter_quant_bias= 0;
522     }else{
523         s->intra_quant_bias=0;
524         s->inter_quant_bias=-(1<<(QUANT_BIAS_SHIFT-2)); //(a - x/4)/x
525     }
526
527     if(avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
528         s->intra_quant_bias= avctx->intra_quant_bias;
529     if(avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
530         s->inter_quant_bias= avctx->inter_quant_bias;
531
532     avcodec_get_chroma_sub_sample(avctx->pix_fmt, &chroma_h_shift, &chroma_v_shift);
533
534     if(avctx->codec_id == CODEC_ID_MPEG4 && s->avctx->time_base.den > (1<<16)-1){
535         av_log(avctx, AV_LOG_ERROR, "timebase %d/%d not supported by MPEG 4 standard, "
536                "the maximum admitted value for the timebase denominator is %d\n",
537                s->avctx->time_base.num, s->avctx->time_base.den, (1<<16)-1);
538         return -1;
539     }
540     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
541
542     switch(avctx->codec->id) {
543     case CODEC_ID_MPEG1VIDEO:
544         s->out_format = FMT_MPEG1;
545         s->low_delay= !!(s->flags & CODEC_FLAG_LOW_DELAY);
546         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
547         break;
548     case CODEC_ID_MPEG2VIDEO:
549         s->out_format = FMT_MPEG1;
550         s->low_delay= !!(s->flags & CODEC_FLAG_LOW_DELAY);
551         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
552         s->rtp_mode= 1;
553         break;
554     case CODEC_ID_LJPEG:
555     case CODEC_ID_MJPEG:
556         s->out_format = FMT_MJPEG;
557         s->intra_only = 1; /* force intra only for jpeg */
558         if(avctx->codec->id == CODEC_ID_LJPEG && avctx->pix_fmt == PIX_FMT_BGRA){
559             s->mjpeg_vsample[0] = s->mjpeg_hsample[0] =
560             s->mjpeg_vsample[1] = s->mjpeg_hsample[1] =
561             s->mjpeg_vsample[2] = s->mjpeg_hsample[2] = 1;
562         }else{
563             s->mjpeg_vsample[0] = 2;
564             s->mjpeg_vsample[1] = 2>>chroma_v_shift;
565             s->mjpeg_vsample[2] = 2>>chroma_v_shift;
566             s->mjpeg_hsample[0] = 2;
567             s->mjpeg_hsample[1] = 2>>chroma_h_shift;
568             s->mjpeg_hsample[2] = 2>>chroma_h_shift;
569         }
570         if (!(CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER)
571             || ff_mjpeg_encode_init(s) < 0)
572             return -1;
573         avctx->delay=0;
574         s->low_delay=1;
575         break;
576     case CODEC_ID_H261:
577         if (!CONFIG_H261_ENCODER)  return -1;
578         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
579             av_log(avctx, AV_LOG_ERROR, "The specified picture size of %dx%d is not valid for the H.261 codec.\nValid sizes are 176x144, 352x288\n", s->width, s->height);
580             return -1;
581         }
582         s->out_format = FMT_H261;
583         avctx->delay=0;
584         s->low_delay=1;
585         break;
586     case CODEC_ID_H263:
587         if (!CONFIG_H263_ENCODER)  return -1;
588         if (ff_match_2uint16(h263_format, FF_ARRAY_ELEMS(h263_format), s->width, s->height) == 8) {
589             av_log(avctx, AV_LOG_INFO, "The specified picture size of %dx%d is not valid for the H.263 codec.\nValid sizes are 128x96, 176x144, 352x288, 704x576, and 1408x1152. Try H.263+.\n", s->width, s->height);
590             return -1;
591         }
592         s->out_format = FMT_H263;
593         s->obmc= (avctx->flags & CODEC_FLAG_OBMC) ? 1:0;
594         avctx->delay=0;
595         s->low_delay=1;
596         break;
597     case CODEC_ID_H263P:
598         s->out_format = FMT_H263;
599         s->h263_plus = 1;
600         /* Fx */
601         s->umvplus = (avctx->flags & CODEC_FLAG_H263P_UMV) ? 1:0;
602         s->h263_aic= (avctx->flags & CODEC_FLAG_AC_PRED) ? 1:0;
603         s->modified_quant= s->h263_aic;
604         s->alt_inter_vlc= (avctx->flags & CODEC_FLAG_H263P_AIV) ? 1:0;
605         s->obmc= (avctx->flags & CODEC_FLAG_OBMC) ? 1:0;
606         s->loop_filter= (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1:0;
607         s->unrestricted_mv= s->obmc || s->loop_filter || s->umvplus;
608         s->h263_slice_structured= (s->flags & CODEC_FLAG_H263P_SLICE_STRUCT) ? 1:0;
609
610         /* /Fx */
611         /* These are just to be sure */
612         avctx->delay=0;
613         s->low_delay=1;
614         break;
615     case CODEC_ID_FLV1:
616         s->out_format = FMT_H263;
617         s->h263_flv = 2; /* format = 1; 11-bit codes */
618         s->unrestricted_mv = 1;
619         s->rtp_mode=0; /* don't allow GOB */
620         avctx->delay=0;
621         s->low_delay=1;
622         break;
623     case CODEC_ID_RV10:
624         s->out_format = FMT_H263;
625         avctx->delay=0;
626         s->low_delay=1;
627         break;
628     case CODEC_ID_RV20:
629         s->out_format = FMT_H263;
630         avctx->delay=0;
631         s->low_delay=1;
632         s->modified_quant=1;
633         s->h263_aic=1;
634         s->h263_plus=1;
635         s->loop_filter=1;
636         s->unrestricted_mv= 0;
637         break;
638     case CODEC_ID_MPEG4:
639         s->out_format = FMT_H263;
640         s->h263_pred = 1;
641         s->unrestricted_mv = 1;
642         s->low_delay= s->max_b_frames ? 0 : 1;
643         avctx->delay= s->low_delay ? 0 : (s->max_b_frames + 1);
644         break;
645     case CODEC_ID_MSMPEG4V2:
646         s->out_format = FMT_H263;
647         s->h263_pred = 1;
648         s->unrestricted_mv = 1;
649         s->msmpeg4_version= 2;
650         avctx->delay=0;
651         s->low_delay=1;
652         break;
653     case CODEC_ID_MSMPEG4V3:
654         s->out_format = FMT_H263;
655         s->h263_pred = 1;
656         s->unrestricted_mv = 1;
657         s->msmpeg4_version= 3;
658         s->flipflop_rounding=1;
659         avctx->delay=0;
660         s->low_delay=1;
661         break;
662     case CODEC_ID_WMV1:
663         s->out_format = FMT_H263;
664         s->h263_pred = 1;
665         s->unrestricted_mv = 1;
666         s->msmpeg4_version= 4;
667         s->flipflop_rounding=1;
668         avctx->delay=0;
669         s->low_delay=1;
670         break;
671     case CODEC_ID_WMV2:
672         s->out_format = FMT_H263;
673         s->h263_pred = 1;
674         s->unrestricted_mv = 1;
675         s->msmpeg4_version= 5;
676         s->flipflop_rounding=1;
677         avctx->delay=0;
678         s->low_delay=1;
679         break;
680     default:
681         return -1;
682     }
683
684     avctx->has_b_frames= !s->low_delay;
685
686     s->encoding = 1;
687
688     s->progressive_frame=
689     s->progressive_sequence= !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT|CODEC_FLAG_INTERLACED_ME|CODEC_FLAG_ALT_SCAN));
690
691     /* init */
692     if (MPV_common_init(s) < 0)
693         return -1;
694
695     if(!s->dct_quantize)
696         s->dct_quantize = dct_quantize_c;
697     if(!s->denoise_dct)
698         s->denoise_dct = denoise_dct_c;
699     s->fast_dct_quantize = s->dct_quantize;
700     if(avctx->trellis)
701         s->dct_quantize = dct_quantize_trellis_c;
702
703     if((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
704         s->chroma_qscale_table= ff_h263_chroma_qscale_table;
705
706     s->quant_precision=5;
707
708     ff_set_cmp(&s->dsp, s->dsp.ildct_cmp, s->avctx->ildct_cmp);
709     ff_set_cmp(&s->dsp, s->dsp.frame_skip_cmp, s->avctx->frame_skip_cmp);
710
711     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
712         ff_h261_encode_init(s);
713     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
714         h263_encode_init(s);
715     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
716         ff_msmpeg4_encode_init(s);
717     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
718         && s->out_format == FMT_MPEG1)
719         ff_mpeg1_encode_init(s);
720
721     /* init q matrix */
722     for(i=0;i<64;i++) {
723         int j= s->dsp.idct_permutation[i];
724         if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4 && s->mpeg_quant){
725             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
726             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
727         }else if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
728             s->intra_matrix[j] =
729             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
730         }else
731         { /* mpeg1/2 */
732             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
733             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
734         }
735         if(s->avctx->intra_matrix)
736             s->intra_matrix[j] = s->avctx->intra_matrix[i];
737         if(s->avctx->inter_matrix)
738             s->inter_matrix[j] = s->avctx->inter_matrix[i];
739     }
740
741     /* precompute matrix */
742     /* for mjpeg, we do include qscale in the matrix */
743     if (s->out_format != FMT_MJPEG) {
744         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
745                        s->intra_matrix, s->intra_quant_bias, avctx->qmin, 31, 1);
746         ff_convert_matrix(&s->dsp, s->q_inter_matrix, s->q_inter_matrix16,
747                        s->inter_matrix, s->inter_quant_bias, avctx->qmin, 31, 0);
748     }
749
750     if(ff_rate_control_init(s) < 0)
751         return -1;
752
753     return 0;
754 }
755
756 av_cold int MPV_encode_end(AVCodecContext *avctx)
757 {
758     MpegEncContext *s = avctx->priv_data;
759
760     ff_rate_control_uninit(s);
761
762     MPV_common_end(s);
763     if ((CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER) && s->out_format == FMT_MJPEG)
764         ff_mjpeg_encode_close(s);
765
766     av_freep(&avctx->extradata);
767
768     return 0;
769 }
770
771 static int get_sae(uint8_t *src, int ref, int stride){
772     int x,y;
773     int acc=0;
774
775     for(y=0; y<16; y++){
776         for(x=0; x<16; x++){
777             acc+= FFABS(src[x+y*stride] - ref);
778         }
779     }
780
781     return acc;
782 }
783
784 static int get_intra_count(MpegEncContext *s, uint8_t *src, uint8_t *ref, int stride){
785     int x, y, w, h;
786     int acc=0;
787
788     w= s->width &~15;
789     h= s->height&~15;
790
791     for(y=0; y<h; y+=16){
792         for(x=0; x<w; x+=16){
793             int offset= x + y*stride;
794             int sad = s->dsp.sad[0](NULL, src + offset, ref + offset, stride, 16);
795             int mean= (s->dsp.pix_sum(src + offset, stride) + 128)>>8;
796             int sae = get_sae(src + offset, mean, stride);
797
798             acc+= sae + 500 < sad;
799         }
800     }
801     return acc;
802 }
803
804
805 static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg){
806     AVFrame *pic=NULL;
807     int64_t pts;
808     int i;
809     const int encoding_delay= s->max_b_frames;
810     int direct=1;
811
812     if(pic_arg){
813         pts= pic_arg->pts;
814         pic_arg->display_picture_number= s->input_picture_number++;
815
816         if(pts != AV_NOPTS_VALUE){
817             if(s->user_specified_pts != AV_NOPTS_VALUE){
818                 int64_t time= pts;
819                 int64_t last= s->user_specified_pts;
820
821                 if(time <= last){
822                     av_log(s->avctx, AV_LOG_ERROR, "Error, Invalid timestamp=%"PRId64", last=%"PRId64"\n", pts, s->user_specified_pts);
823                     return -1;
824                 }
825             }
826             s->user_specified_pts= pts;
827         }else{
828             if(s->user_specified_pts != AV_NOPTS_VALUE){
829                 s->user_specified_pts=
830                 pts= s->user_specified_pts + 1;
831                 av_log(s->avctx, AV_LOG_INFO, "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n", pts);
832             }else{
833                 pts= pic_arg->display_picture_number;
834             }
835         }
836     }
837
838   if(pic_arg){
839     if(encoding_delay && !(s->flags&CODEC_FLAG_INPUT_PRESERVED)) direct=0;
840     if(pic_arg->linesize[0] != s->linesize) direct=0;
841     if(pic_arg->linesize[1] != s->uvlinesize) direct=0;
842     if(pic_arg->linesize[2] != s->uvlinesize) direct=0;
843
844 //    av_log(AV_LOG_DEBUG, "%d %d %d %d\n",pic_arg->linesize[0], pic_arg->linesize[1], s->linesize, s->uvlinesize);
845
846     if(direct){
847         i= ff_find_unused_picture(s, 1);
848
849         pic= (AVFrame*)&s->picture[i];
850         pic->reference= 3;
851
852         for(i=0; i<4; i++){
853             pic->data[i]= pic_arg->data[i];
854             pic->linesize[i]= pic_arg->linesize[i];
855         }
856         if(ff_alloc_picture(s, (Picture*)pic, 1) < 0){
857             return -1;
858         }
859     }else{
860         i= ff_find_unused_picture(s, 0);
861
862         pic= (AVFrame*)&s->picture[i];
863         pic->reference= 3;
864
865         if(ff_alloc_picture(s, (Picture*)pic, 0) < 0){
866             return -1;
867         }
868
869         if(   pic->data[0] + INPLACE_OFFSET == pic_arg->data[0]
870            && pic->data[1] + INPLACE_OFFSET == pic_arg->data[1]
871            && pic->data[2] + INPLACE_OFFSET == pic_arg->data[2]){
872        // empty
873         }else{
874             int h_chroma_shift, v_chroma_shift;
875             avcodec_get_chroma_sub_sample(s->avctx->pix_fmt, &h_chroma_shift, &v_chroma_shift);
876
877             for(i=0; i<3; i++){
878                 int src_stride= pic_arg->linesize[i];
879                 int dst_stride= i ? s->uvlinesize : s->linesize;
880                 int h_shift= i ? h_chroma_shift : 0;
881                 int v_shift= i ? v_chroma_shift : 0;
882                 int w= s->width >>h_shift;
883                 int h= s->height>>v_shift;
884                 uint8_t *src= pic_arg->data[i];
885                 uint8_t *dst= pic->data[i];
886
887                 if(!s->avctx->rc_buffer_size)
888                     dst +=INPLACE_OFFSET;
889
890                 if(src_stride==dst_stride)
891                     memcpy(dst, src, src_stride*h);
892                 else{
893                     while(h--){
894                         memcpy(dst, src, w);
895                         dst += dst_stride;
896                         src += src_stride;
897                     }
898                 }
899             }
900         }
901     }
902     copy_picture_attributes(s, pic, pic_arg);
903     pic->pts= pts; //we set this here to avoid modifiying pic_arg
904   }
905
906     /* shift buffer entries */
907     for(i=1; i<MAX_PICTURE_COUNT /*s->encoding_delay+1*/; i++)
908         s->input_picture[i-1]= s->input_picture[i];
909
910     s->input_picture[encoding_delay]= (Picture*)pic;
911
912     return 0;
913 }
914
915 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref){
916     int x, y, plane;
917     int score=0;
918     int64_t score64=0;
919
920     for(plane=0; plane<3; plane++){
921         const int stride = p->f.linesize[plane];
922         const int bw= plane ? 1 : 2;
923         for(y=0; y<s->mb_height*bw; y++){
924             for(x=0; x<s->mb_width*bw; x++){
925                 int off = p->f.type == FF_BUFFER_TYPE_SHARED ? 0: 16;
926                 int v   = s->dsp.frame_skip_cmp[1](s, p->f.data[plane] + 8*(x + y*stride)+off, ref->f.data[plane] + 8*(x + y*stride), stride, 8);
927
928                 switch(s->avctx->frame_skip_exp){
929                     case 0: score= FFMAX(score, v); break;
930                     case 1: score+= FFABS(v);break;
931                     case 2: score+= v*v;break;
932                     case 3: score64+= FFABS(v*v*(int64_t)v);break;
933                     case 4: score64+= v*v*(int64_t)(v*v);break;
934                 }
935             }
936         }
937     }
938
939     if(score) score64= score;
940
941     if(score64 < s->avctx->frame_skip_threshold)
942         return 1;
943     if(score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda)>>8))
944         return 1;
945     return 0;
946 }
947
948 static int estimate_best_b_count(MpegEncContext *s){
949     AVCodec *codec= avcodec_find_encoder(s->avctx->codec_id);
950     AVCodecContext *c = avcodec_alloc_context3(NULL);
951     AVFrame input[FF_MAX_B_FRAMES+2];
952     const int scale= s->avctx->brd_scale;
953     int i, j, out_size, p_lambda, b_lambda, lambda2;
954     int outbuf_size= s->width * s->height; //FIXME
955     uint8_t *outbuf= av_malloc(outbuf_size);
956     int64_t best_rd= INT64_MAX;
957     int best_b_count= -1;
958
959     assert(scale>=0 && scale <=3);
960
961 //    emms_c();
962     p_lambda= s->last_lambda_for[AV_PICTURE_TYPE_P]; //s->next_picture_ptr->quality;
963     b_lambda= s->last_lambda_for[AV_PICTURE_TYPE_B]; //p_lambda *FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
964     if(!b_lambda) b_lambda= p_lambda; //FIXME we should do this somewhere else
965     lambda2= (b_lambda*b_lambda + (1<<FF_LAMBDA_SHIFT)/2 ) >> FF_LAMBDA_SHIFT;
966
967     c->width = s->width >> scale;
968     c->height= s->height>> scale;
969     c->flags= CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR | CODEC_FLAG_INPUT_PRESERVED /*| CODEC_FLAG_EMU_EDGE*/;
970     c->flags|= s->avctx->flags & CODEC_FLAG_QPEL;
971     c->mb_decision= s->avctx->mb_decision;
972     c->me_cmp= s->avctx->me_cmp;
973     c->mb_cmp= s->avctx->mb_cmp;
974     c->me_sub_cmp= s->avctx->me_sub_cmp;
975     c->pix_fmt = PIX_FMT_YUV420P;
976     c->time_base= s->avctx->time_base;
977     c->max_b_frames= s->max_b_frames;
978
979     if (avcodec_open2(c, codec, NULL) < 0)
980         return -1;
981
982     for(i=0; i<s->max_b_frames+2; i++){
983         int ysize= c->width*c->height;
984         int csize= (c->width/2)*(c->height/2);
985         Picture pre_input, *pre_input_ptr= i ? s->input_picture[i-1] : s->next_picture_ptr;
986
987         avcodec_get_frame_defaults(&input[i]);
988         input[i].data[0]= av_malloc(ysize + 2*csize);
989         input[i].data[1]= input[i].data[0] + ysize;
990         input[i].data[2]= input[i].data[1] + csize;
991         input[i].linesize[0]= c->width;
992         input[i].linesize[1]=
993         input[i].linesize[2]= c->width/2;
994
995         if(pre_input_ptr && (!i || s->input_picture[i-1])) {
996             pre_input= *pre_input_ptr;
997
998             if (pre_input.f.type != FF_BUFFER_TYPE_SHARED && i) {
999                 pre_input.f.data[0] += INPLACE_OFFSET;
1000                 pre_input.f.data[1] += INPLACE_OFFSET;
1001                 pre_input.f.data[2] += INPLACE_OFFSET;
1002             }
1003
1004             s->dsp.shrink[scale](input[i].data[0], input[i].linesize[0], pre_input.f.data[0], pre_input.f.linesize[0], c->width,      c->height);
1005             s->dsp.shrink[scale](input[i].data[1], input[i].linesize[1], pre_input.f.data[1], pre_input.f.linesize[1], c->width >> 1, c->height >> 1);
1006             s->dsp.shrink[scale](input[i].data[2], input[i].linesize[2], pre_input.f.data[2], pre_input.f.linesize[2], c->width >> 1, c->height >> 1);
1007         }
1008     }
1009
1010     for(j=0; j<s->max_b_frames+1; j++){
1011         int64_t rd=0;
1012
1013         if(!s->input_picture[j])
1014             break;
1015
1016         c->error[0]= c->error[1]= c->error[2]= 0;
1017
1018         input[0].pict_type= AV_PICTURE_TYPE_I;
1019         input[0].quality= 1 * FF_QP2LAMBDA;
1020         out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[0]);
1021 //        rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1022
1023         for(i=0; i<s->max_b_frames+1; i++){
1024             int is_p= i % (j+1) == j || i==s->max_b_frames;
1025
1026             input[i+1].pict_type= is_p ? AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1027             input[i+1].quality= is_p ? p_lambda : b_lambda;
1028             out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[i+1]);
1029             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1030         }
1031
1032         /* get the delayed frames */
1033         while(out_size){
1034             out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL);
1035             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1036         }
1037
1038         rd += c->error[0] + c->error[1] + c->error[2];
1039
1040         if(rd < best_rd){
1041             best_rd= rd;
1042             best_b_count= j;
1043         }
1044     }
1045
1046     av_freep(&outbuf);
1047     avcodec_close(c);
1048     av_freep(&c);
1049
1050     for(i=0; i<s->max_b_frames+2; i++){
1051         av_freep(&input[i].data[0]);
1052     }
1053
1054     return best_b_count;
1055 }
1056
1057 static int select_input_picture(MpegEncContext *s){
1058     int i;
1059
1060     for(i=1; i<MAX_PICTURE_COUNT; i++)
1061         s->reordered_input_picture[i-1]= s->reordered_input_picture[i];
1062     s->reordered_input_picture[MAX_PICTURE_COUNT-1]= NULL;
1063
1064     /* set next picture type & ordering */
1065     if(s->reordered_input_picture[0]==NULL && s->input_picture[0]){
1066         if(/*s->picture_in_gop_number >= s->gop_size ||*/ s->next_picture_ptr==NULL || s->intra_only){
1067             s->reordered_input_picture[0]= s->input_picture[0];
1068             s->reordered_input_picture[0]->f.pict_type = AV_PICTURE_TYPE_I;
1069             s->reordered_input_picture[0]->f.coded_picture_number = s->coded_picture_number++;
1070         }else{
1071             int b_frames;
1072
1073             if(s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor){
1074                 if(s->picture_in_gop_number < s->gop_size && skip_check(s, s->input_picture[0], s->next_picture_ptr)){
1075                 //FIXME check that te gop check above is +-1 correct
1076 //av_log(NULL, AV_LOG_DEBUG, "skip %p %"PRId64"\n", s->input_picture[0]->f.data[0], s->input_picture[0]->pts);
1077
1078                     if (s->input_picture[0]->f.type == FF_BUFFER_TYPE_SHARED) {
1079                         for(i=0; i<4; i++)
1080                             s->input_picture[0]->f.data[i] = NULL;
1081                         s->input_picture[0]->f.type = 0;
1082                     }else{
1083                         assert(   s->input_picture[0]->type==FF_BUFFER_TYPE_USER
1084                                || s->input_picture[0]->type==FF_BUFFER_TYPE_INTERNAL);
1085
1086                         s->avctx->release_buffer(s->avctx, (AVFrame*)s->input_picture[0]);
1087                     }
1088
1089                     emms_c();
1090                     ff_vbv_update(s, 0);
1091
1092                     goto no_output_pic;
1093                 }
1094             }
1095
1096             if(s->flags&CODEC_FLAG_PASS2){
1097                 for(i=0; i<s->max_b_frames+1; i++){
1098                     int pict_num = s->input_picture[0]->f.display_picture_number + i;
1099
1100                     if(pict_num >= s->rc_context.num_entries)
1101                         break;
1102                     if(!s->input_picture[i]){
1103                         s->rc_context.entry[pict_num-1].new_pict_type = AV_PICTURE_TYPE_P;
1104                         break;
1105                     }
1106
1107                     s->input_picture[i]->f.pict_type =
1108                         s->rc_context.entry[pict_num].new_pict_type;
1109                 }
1110             }
1111
1112             if(s->avctx->b_frame_strategy==0){
1113                 b_frames= s->max_b_frames;
1114                 while(b_frames && !s->input_picture[b_frames]) b_frames--;
1115             }else if(s->avctx->b_frame_strategy==1){
1116                 for(i=1; i<s->max_b_frames+1; i++){
1117                     if(s->input_picture[i] && s->input_picture[i]->b_frame_score==0){
1118                         s->input_picture[i]->b_frame_score=
1119                             get_intra_count(s, s->input_picture[i  ]->f.data[0],
1120                                                s->input_picture[i-1]->f.data[0], s->linesize) + 1;
1121                     }
1122                 }
1123                 for(i=0; i<s->max_b_frames+1; i++){
1124                     if(s->input_picture[i]==NULL || s->input_picture[i]->b_frame_score - 1 > s->mb_num/s->avctx->b_sensitivity) break;
1125                 }
1126
1127                 b_frames= FFMAX(0, i-1);
1128
1129                 /* reset scores */
1130                 for(i=0; i<b_frames+1; i++){
1131                     s->input_picture[i]->b_frame_score=0;
1132                 }
1133             }else if(s->avctx->b_frame_strategy==2){
1134                 b_frames= estimate_best_b_count(s);
1135             }else{
1136                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1137                 b_frames=0;
1138             }
1139
1140             emms_c();
1141 //static int b_count=0;
1142 //b_count+= b_frames;
1143 //av_log(s->avctx, AV_LOG_DEBUG, "b_frames: %d\n", b_count);
1144
1145             for(i= b_frames - 1; i>=0; i--){
1146                 int type = s->input_picture[i]->f.pict_type;
1147                 if(type && type != AV_PICTURE_TYPE_B)
1148                     b_frames= i;
1149             }
1150             if (s->input_picture[b_frames]->f.pict_type == AV_PICTURE_TYPE_B && b_frames == s->max_b_frames){
1151                 av_log(s->avctx, AV_LOG_ERROR, "warning, too many b frames in a row\n");
1152             }
1153
1154             if(s->picture_in_gop_number + b_frames >= s->gop_size){
1155               if((s->flags2 & CODEC_FLAG2_STRICT_GOP) && s->gop_size > s->picture_in_gop_number){
1156                     b_frames= s->gop_size - s->picture_in_gop_number - 1;
1157               }else{
1158                 if(s->flags & CODEC_FLAG_CLOSED_GOP)
1159                     b_frames=0;
1160                 s->input_picture[b_frames]->f.pict_type = AV_PICTURE_TYPE_I;
1161               }
1162             }
1163
1164             if(   (s->flags & CODEC_FLAG_CLOSED_GOP)
1165                && b_frames
1166                && s->input_picture[b_frames]->f.pict_type== AV_PICTURE_TYPE_I)
1167                 b_frames--;
1168
1169             s->reordered_input_picture[0]= s->input_picture[b_frames];
1170             if (s->reordered_input_picture[0]->f.pict_type != AV_PICTURE_TYPE_I)
1171                 s->reordered_input_picture[0]->f.pict_type = AV_PICTURE_TYPE_P;
1172             s->reordered_input_picture[0]->f.coded_picture_number = s->coded_picture_number++;
1173             for(i=0; i<b_frames; i++){
1174                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1175                 s->reordered_input_picture[i + 1]->f.pict_type = AV_PICTURE_TYPE_B;
1176                 s->reordered_input_picture[i + 1]->f.coded_picture_number = s->coded_picture_number++;
1177             }
1178         }
1179     }
1180 no_output_pic:
1181     if(s->reordered_input_picture[0]){
1182         s->reordered_input_picture[0]->f.reference = s->reordered_input_picture[0]->f.pict_type!=AV_PICTURE_TYPE_B ? 3 : 0;
1183
1184         ff_copy_picture(&s->new_picture, s->reordered_input_picture[0]);
1185
1186         if (s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_SHARED || s->avctx->rc_buffer_size) {
1187             // input is a shared pix, so we can't modifiy it -> alloc a new one & ensure that the shared one is reuseable
1188
1189             int i= ff_find_unused_picture(s, 0);
1190             Picture *pic= &s->picture[i];
1191
1192             pic->f.reference = s->reordered_input_picture[0]->f.reference;
1193             if(ff_alloc_picture(s, pic, 0) < 0){
1194                 return -1;
1195             }
1196
1197             /* mark us unused / free shared pic */
1198             if (s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_INTERNAL)
1199                 s->avctx->release_buffer(s->avctx, (AVFrame*)s->reordered_input_picture[0]);
1200             for(i=0; i<4; i++)
1201                 s->reordered_input_picture[0]->f.data[i] = NULL;
1202             s->reordered_input_picture[0]->f.type = 0;
1203
1204             copy_picture_attributes(s, (AVFrame*)pic, (AVFrame*)s->reordered_input_picture[0]);
1205
1206             s->current_picture_ptr= pic;
1207         }else{
1208             // input is not a shared pix -> reuse buffer for current_pix
1209
1210             assert(   s->reordered_input_picture[0]->type==FF_BUFFER_TYPE_USER
1211                    || s->reordered_input_picture[0]->type==FF_BUFFER_TYPE_INTERNAL);
1212
1213             s->current_picture_ptr= s->reordered_input_picture[0];
1214             for(i=0; i<4; i++){
1215                 s->new_picture.f.data[i] += INPLACE_OFFSET;
1216             }
1217         }
1218         ff_copy_picture(&s->current_picture, s->current_picture_ptr);
1219
1220         s->picture_number = s->new_picture.f.display_picture_number;
1221 //printf("dpn:%d\n", s->picture_number);
1222     }else{
1223        memset(&s->new_picture, 0, sizeof(Picture));
1224     }
1225     return 0;
1226 }
1227
1228 int MPV_encode_picture(AVCodecContext *avctx,
1229                        unsigned char *buf, int buf_size, void *data)
1230 {
1231     MpegEncContext *s = avctx->priv_data;
1232     AVFrame *pic_arg = data;
1233     int i, stuffing_count, context_count = avctx->thread_count;
1234
1235     for(i=0; i<context_count; i++){
1236         int start_y= s->thread_context[i]->start_mb_y;
1237         int   end_y= s->thread_context[i]->  end_mb_y;
1238         int h= s->mb_height;
1239         uint8_t *start= buf + (size_t)(((int64_t) buf_size)*start_y/h);
1240         uint8_t *end  = buf + (size_t)(((int64_t) buf_size)*  end_y/h);
1241
1242         init_put_bits(&s->thread_context[i]->pb, start, end - start);
1243     }
1244
1245     s->picture_in_gop_number++;
1246
1247     if(load_input_picture(s, pic_arg) < 0)
1248         return -1;
1249
1250     if(select_input_picture(s) < 0){
1251         return -1;
1252     }
1253
1254     /* output? */
1255     if (s->new_picture.f.data[0]) {
1256         s->pict_type = s->new_picture.f.pict_type;
1257 //emms_c();
1258 //printf("qs:%f %f %d\n", s->new_picture.quality, s->current_picture.quality, s->qscale);
1259         MPV_frame_start(s, avctx);
1260 vbv_retry:
1261         if (encode_picture(s, s->picture_number) < 0)
1262             return -1;
1263
1264         avctx->header_bits = s->header_bits;
1265         avctx->mv_bits     = s->mv_bits;
1266         avctx->misc_bits   = s->misc_bits;
1267         avctx->i_tex_bits  = s->i_tex_bits;
1268         avctx->p_tex_bits  = s->p_tex_bits;
1269         avctx->i_count     = s->i_count;
1270         avctx->p_count     = s->mb_num - s->i_count - s->skip_count; //FIXME f/b_count in avctx
1271         avctx->skip_count  = s->skip_count;
1272
1273         MPV_frame_end(s);
1274
1275         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1276             ff_mjpeg_encode_picture_trailer(s);
1277
1278         if(avctx->rc_buffer_size){
1279             RateControlContext *rcc= &s->rc_context;
1280             int max_size= rcc->buffer_index * avctx->rc_max_available_vbv_use;
1281
1282             if(put_bits_count(&s->pb) > max_size && s->lambda < s->avctx->lmax){
1283                 s->next_lambda= FFMAX(s->lambda+1, s->lambda*(s->qscale+1) / s->qscale);
1284                 if(s->adaptive_quant){
1285                     int i;
1286                     for(i=0; i<s->mb_height*s->mb_stride; i++)
1287                         s->lambda_table[i]= FFMAX(s->lambda_table[i]+1, s->lambda_table[i]*(s->qscale+1) / s->qscale);
1288                 }
1289                 s->mb_skipped = 0;        //done in MPV_frame_start()
1290                 if(s->pict_type==AV_PICTURE_TYPE_P){ //done in encode_picture() so we must undo it
1291                     if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
1292                         s->no_rounding ^= 1;
1293                 }
1294                 if(s->pict_type!=AV_PICTURE_TYPE_B){
1295                     s->time_base= s->last_time_base;
1296                     s->last_non_b_time= s->time - s->pp_time;
1297                 }
1298 //                av_log(NULL, AV_LOG_ERROR, "R:%d ", s->next_lambda);
1299                 for(i=0; i<context_count; i++){
1300                     PutBitContext *pb= &s->thread_context[i]->pb;
1301                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1302                 }
1303                 goto vbv_retry;
1304             }
1305
1306             assert(s->avctx->rc_max_rate);
1307         }
1308
1309         if(s->flags&CODEC_FLAG_PASS1)
1310             ff_write_pass1_stats(s);
1311
1312         for(i=0; i<4; i++){
1313             s->current_picture_ptr->f.error[i]  = s->current_picture.f.error[i];
1314             avctx->error[i]                        += s->current_picture_ptr->f.error[i];
1315         }
1316
1317         if(s->flags&CODEC_FLAG_PASS1)
1318             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits + avctx->i_tex_bits + avctx->p_tex_bits == put_bits_count(&s->pb));
1319         flush_put_bits(&s->pb);
1320         s->frame_bits  = put_bits_count(&s->pb);
1321
1322         stuffing_count= ff_vbv_update(s, s->frame_bits);
1323         if(stuffing_count){
1324             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < stuffing_count + 50){
1325                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1326                 return -1;
1327             }
1328
1329             switch(s->codec_id){
1330             case CODEC_ID_MPEG1VIDEO:
1331             case CODEC_ID_MPEG2VIDEO:
1332                 while(stuffing_count--){
1333                     put_bits(&s->pb, 8, 0);
1334                 }
1335             break;
1336             case CODEC_ID_MPEG4:
1337                 put_bits(&s->pb, 16, 0);
1338                 put_bits(&s->pb, 16, 0x1C3);
1339                 stuffing_count -= 4;
1340                 while(stuffing_count--){
1341                     put_bits(&s->pb, 8, 0xFF);
1342                 }
1343             break;
1344             default:
1345                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1346             }
1347             flush_put_bits(&s->pb);
1348             s->frame_bits  = put_bits_count(&s->pb);
1349         }
1350
1351         /* update mpeg1/2 vbv_delay for CBR */
1352         if(s->avctx->rc_max_rate && s->avctx->rc_min_rate == s->avctx->rc_max_rate && s->out_format == FMT_MPEG1
1353            && 90000LL * (avctx->rc_buffer_size-1) <= s->avctx->rc_max_rate*0xFFFFLL){
1354             int vbv_delay, min_delay;
1355             double inbits = s->avctx->rc_max_rate*av_q2d(s->avctx->time_base);
1356             int    minbits= s->frame_bits - 8*(s->vbv_delay_ptr - s->pb.buf - 1);
1357             double bits   = s->rc_context.buffer_index + minbits - inbits;
1358
1359             if(bits<0)
1360                 av_log(s->avctx, AV_LOG_ERROR, "Internal error, negative bits\n");
1361
1362             assert(s->repeat_first_field==0);
1363
1364             vbv_delay=     bits * 90000                               / s->avctx->rc_max_rate;
1365             min_delay= (minbits * 90000LL + s->avctx->rc_max_rate - 1)/ s->avctx->rc_max_rate;
1366
1367             vbv_delay= FFMAX(vbv_delay, min_delay);
1368
1369             assert(vbv_delay < 0xFFFF);
1370
1371             s->vbv_delay_ptr[0] &= 0xF8;
1372             s->vbv_delay_ptr[0] |= vbv_delay>>13;
1373             s->vbv_delay_ptr[1]  = vbv_delay>>5;
1374             s->vbv_delay_ptr[2] &= 0x07;
1375             s->vbv_delay_ptr[2] |= vbv_delay<<3;
1376             avctx->vbv_delay = vbv_delay*300;
1377         }
1378         s->total_bits += s->frame_bits;
1379         avctx->frame_bits  = s->frame_bits;
1380     }else{
1381         assert((put_bits_ptr(&s->pb) == s->pb.buf));
1382         s->frame_bits=0;
1383     }
1384     assert((s->frame_bits&7)==0);
1385
1386     return s->frame_bits/8;
1387 }
1388
1389 static inline void dct_single_coeff_elimination(MpegEncContext *s, int n, int threshold)
1390 {
1391     static const char tab[64]=
1392         {3,2,2,1,1,1,1,1,
1393          1,1,1,1,1,1,1,1,
1394          1,1,1,1,1,1,1,1,
1395          0,0,0,0,0,0,0,0,
1396          0,0,0,0,0,0,0,0,
1397          0,0,0,0,0,0,0,0,
1398          0,0,0,0,0,0,0,0,
1399          0,0,0,0,0,0,0,0};
1400     int score=0;
1401     int run=0;
1402     int i;
1403     DCTELEM *block= s->block[n];
1404     const int last_index= s->block_last_index[n];
1405     int skip_dc;
1406
1407     if(threshold<0){
1408         skip_dc=0;
1409         threshold= -threshold;
1410     }else
1411         skip_dc=1;
1412
1413     /* Are all we could set to zero already zero? */
1414     if(last_index<=skip_dc - 1) return;
1415
1416     for(i=0; i<=last_index; i++){
1417         const int j = s->intra_scantable.permutated[i];
1418         const int level = FFABS(block[j]);
1419         if(level==1){
1420             if(skip_dc && i==0) continue;
1421             score+= tab[run];
1422             run=0;
1423         }else if(level>1){
1424             return;
1425         }else{
1426             run++;
1427         }
1428     }
1429     if(score >= threshold) return;
1430     for(i=skip_dc; i<=last_index; i++){
1431         const int j = s->intra_scantable.permutated[i];
1432         block[j]=0;
1433     }
1434     if(block[0]) s->block_last_index[n]= 0;
1435     else         s->block_last_index[n]= -1;
1436 }
1437
1438 static inline void clip_coeffs(MpegEncContext *s, DCTELEM *block, int last_index)
1439 {
1440     int i;
1441     const int maxlevel= s->max_qcoeff;
1442     const int minlevel= s->min_qcoeff;
1443     int overflow=0;
1444
1445     if(s->mb_intra){
1446         i=1; //skip clipping of intra dc
1447     }else
1448         i=0;
1449
1450     for(;i<=last_index; i++){
1451         const int j= s->intra_scantable.permutated[i];
1452         int level = block[j];
1453
1454         if     (level>maxlevel){
1455             level=maxlevel;
1456             overflow++;
1457         }else if(level<minlevel){
1458             level=minlevel;
1459             overflow++;
1460         }
1461
1462         block[j]= level;
1463     }
1464
1465     if(overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1466         av_log(s->avctx, AV_LOG_INFO, "warning, clipping %d dct coefficients to %d..%d\n", overflow, minlevel, maxlevel);
1467 }
1468
1469 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride){
1470     int x, y;
1471 //FIXME optimize
1472     for(y=0; y<8; y++){
1473         for(x=0; x<8; x++){
1474             int x2, y2;
1475             int sum=0;
1476             int sqr=0;
1477             int count=0;
1478
1479             for(y2= FFMAX(y-1, 0); y2 < FFMIN(8, y+2); y2++){
1480                 for(x2= FFMAX(x-1, 0); x2 < FFMIN(8, x+2); x2++){
1481                     int v= ptr[x2 + y2*stride];
1482                     sum += v;
1483                     sqr += v*v;
1484                     count++;
1485                 }
1486             }
1487             weight[x + 8*y]= (36*ff_sqrt(count*sqr - sum*sum)) / count;
1488         }
1489     }
1490 }
1491
1492 static av_always_inline void encode_mb_internal(MpegEncContext *s, int motion_x, int motion_y, int mb_block_height, int mb_block_count)
1493 {
1494     int16_t weight[8][64];
1495     DCTELEM orig[8][64];
1496     const int mb_x= s->mb_x;
1497     const int mb_y= s->mb_y;
1498     int i;
1499     int skip_dct[8];
1500     int dct_offset   = s->linesize*8; //default for progressive frames
1501     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1502     int wrap_y, wrap_c;
1503
1504     for(i=0; i<mb_block_count; i++) skip_dct[i]=s->skipdct;
1505
1506     if(s->adaptive_quant){
1507         const int last_qp= s->qscale;
1508         const int mb_xy= mb_x + mb_y*s->mb_stride;
1509
1510         s->lambda= s->lambda_table[mb_xy];
1511         update_qscale(s);
1512
1513         if(!(s->flags&CODEC_FLAG_QP_RD)){
1514             s->qscale = s->current_picture_ptr->f.qscale_table[mb_xy];
1515             s->dquant= s->qscale - last_qp;
1516
1517             if(s->out_format==FMT_H263){
1518                 s->dquant= av_clip(s->dquant, -2, 2);
1519
1520                 if(s->codec_id==CODEC_ID_MPEG4){
1521                     if(!s->mb_intra){
1522                         if(s->pict_type == AV_PICTURE_TYPE_B){
1523                             if(s->dquant&1 || s->mv_dir&MV_DIRECT)
1524                                 s->dquant= 0;
1525                         }
1526                         if(s->mv_type==MV_TYPE_8X8)
1527                             s->dquant=0;
1528                     }
1529                 }
1530             }
1531         }
1532         ff_set_qscale(s, last_qp + s->dquant);
1533     }else if(s->flags&CODEC_FLAG_QP_RD)
1534         ff_set_qscale(s, s->qscale + s->dquant);
1535
1536     wrap_y = s->linesize;
1537     wrap_c = s->uvlinesize;
1538     ptr_y  = s->new_picture.f.data[0] + (mb_y * 16 * wrap_y) + mb_x * 16;
1539     ptr_cb = s->new_picture.f.data[1] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1540     ptr_cr = s->new_picture.f.data[2] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1541
1542     if(mb_x*16+16 > s->width || mb_y*16+16 > s->height){
1543         uint8_t *ebuf= s->edge_emu_buffer + 32;
1544         s->dsp.emulated_edge_mc(ebuf            , ptr_y , wrap_y,16,16,mb_x*16,mb_y*16, s->width   , s->height);
1545         ptr_y= ebuf;
1546         s->dsp.emulated_edge_mc(ebuf+18*wrap_y  , ptr_cb, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1547         ptr_cb= ebuf+18*wrap_y;
1548         s->dsp.emulated_edge_mc(ebuf+18*wrap_y+8, ptr_cr, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1549         ptr_cr= ebuf+18*wrap_y+8;
1550     }
1551
1552     if (s->mb_intra) {
1553         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1554             int progressive_score, interlaced_score;
1555
1556             s->interlaced_dct=0;
1557             progressive_score= s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y, 8)
1558                               +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y*8, NULL, wrap_y, 8) - 400;
1559
1560             if(progressive_score > 0){
1561                 interlaced_score = s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y*2, 8)
1562                                   +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y  , NULL, wrap_y*2, 8);
1563                 if(progressive_score > interlaced_score){
1564                     s->interlaced_dct=1;
1565
1566                     dct_offset= wrap_y;
1567                     wrap_y<<=1;
1568                     if (s->chroma_format == CHROMA_422)
1569                         wrap_c<<=1;
1570                 }
1571             }
1572         }
1573
1574         s->dsp.get_pixels(s->block[0], ptr_y                 , wrap_y);
1575         s->dsp.get_pixels(s->block[1], ptr_y              + 8, wrap_y);
1576         s->dsp.get_pixels(s->block[2], ptr_y + dct_offset    , wrap_y);
1577         s->dsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
1578
1579         if(s->flags&CODEC_FLAG_GRAY){
1580             skip_dct[4]= 1;
1581             skip_dct[5]= 1;
1582         }else{
1583             s->dsp.get_pixels(s->block[4], ptr_cb, wrap_c);
1584             s->dsp.get_pixels(s->block[5], ptr_cr, wrap_c);
1585             if(!s->chroma_y_shift){ /* 422 */
1586                 s->dsp.get_pixels(s->block[6], ptr_cb + (dct_offset>>1), wrap_c);
1587                 s->dsp.get_pixels(s->block[7], ptr_cr + (dct_offset>>1), wrap_c);
1588             }
1589         }
1590     }else{
1591         op_pixels_func (*op_pix)[4];
1592         qpel_mc_func (*op_qpix)[16];
1593         uint8_t *dest_y, *dest_cb, *dest_cr;
1594
1595         dest_y  = s->dest[0];
1596         dest_cb = s->dest[1];
1597         dest_cr = s->dest[2];
1598
1599         if ((!s->no_rounding) || s->pict_type==AV_PICTURE_TYPE_B){
1600             op_pix = s->dsp.put_pixels_tab;
1601             op_qpix= s->dsp.put_qpel_pixels_tab;
1602         }else{
1603             op_pix = s->dsp.put_no_rnd_pixels_tab;
1604             op_qpix= s->dsp.put_no_rnd_qpel_pixels_tab;
1605         }
1606
1607         if (s->mv_dir & MV_DIR_FORWARD) {
1608             MPV_motion(s, dest_y, dest_cb, dest_cr, 0, s->last_picture.f.data, op_pix, op_qpix);
1609             op_pix = s->dsp.avg_pixels_tab;
1610             op_qpix= s->dsp.avg_qpel_pixels_tab;
1611         }
1612         if (s->mv_dir & MV_DIR_BACKWARD) {
1613             MPV_motion(s, dest_y, dest_cb, dest_cr, 1, s->next_picture.f.data, op_pix, op_qpix);
1614         }
1615
1616         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1617             int progressive_score, interlaced_score;
1618
1619             s->interlaced_dct=0;
1620             progressive_score= s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y, 8)
1621                               +s->dsp.ildct_cmp[0](s, dest_y + wrap_y*8, ptr_y + wrap_y*8, wrap_y, 8) - 400;
1622
1623             if(s->avctx->ildct_cmp == FF_CMP_VSSE) progressive_score -= 400;
1624
1625             if(progressive_score>0){
1626                 interlaced_score = s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y*2, 8)
1627                                   +s->dsp.ildct_cmp[0](s, dest_y + wrap_y  , ptr_y + wrap_y  , wrap_y*2, 8);
1628
1629                 if(progressive_score > interlaced_score){
1630                     s->interlaced_dct=1;
1631
1632                     dct_offset= wrap_y;
1633                     wrap_y<<=1;
1634                     if (s->chroma_format == CHROMA_422)
1635                         wrap_c<<=1;
1636                 }
1637             }
1638         }
1639
1640         s->dsp.diff_pixels(s->block[0], ptr_y                 , dest_y                 , wrap_y);
1641         s->dsp.diff_pixels(s->block[1], ptr_y              + 8, dest_y              + 8, wrap_y);
1642         s->dsp.diff_pixels(s->block[2], ptr_y + dct_offset    , dest_y + dct_offset    , wrap_y);
1643         s->dsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8, dest_y + dct_offset + 8, wrap_y);
1644
1645         if(s->flags&CODEC_FLAG_GRAY){
1646             skip_dct[4]= 1;
1647             skip_dct[5]= 1;
1648         }else{
1649             s->dsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
1650             s->dsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
1651             if(!s->chroma_y_shift){ /* 422 */
1652                 s->dsp.diff_pixels(s->block[6], ptr_cb + (dct_offset>>1), dest_cb + (dct_offset>>1), wrap_c);
1653                 s->dsp.diff_pixels(s->block[7], ptr_cr + (dct_offset>>1), dest_cr + (dct_offset>>1), wrap_c);
1654             }
1655         }
1656         /* pre quantization */
1657         if(s->current_picture.mc_mb_var[s->mb_stride*mb_y+ mb_x]<2*s->qscale*s->qscale){
1658             //FIXME optimize
1659             if(s->dsp.sad[1](NULL, ptr_y               , dest_y               , wrap_y, 8) < 20*s->qscale) skip_dct[0]= 1;
1660             if(s->dsp.sad[1](NULL, ptr_y            + 8, dest_y            + 8, wrap_y, 8) < 20*s->qscale) skip_dct[1]= 1;
1661             if(s->dsp.sad[1](NULL, ptr_y +dct_offset   , dest_y +dct_offset   , wrap_y, 8) < 20*s->qscale) skip_dct[2]= 1;
1662             if(s->dsp.sad[1](NULL, ptr_y +dct_offset+ 8, dest_y +dct_offset+ 8, wrap_y, 8) < 20*s->qscale) skip_dct[3]= 1;
1663             if(s->dsp.sad[1](NULL, ptr_cb              , dest_cb              , wrap_c, 8) < 20*s->qscale) skip_dct[4]= 1;
1664             if(s->dsp.sad[1](NULL, ptr_cr              , dest_cr              , wrap_c, 8) < 20*s->qscale) skip_dct[5]= 1;
1665             if(!s->chroma_y_shift){ /* 422 */
1666                 if(s->dsp.sad[1](NULL, ptr_cb +(dct_offset>>1), dest_cb +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[6]= 1;
1667                 if(s->dsp.sad[1](NULL, ptr_cr +(dct_offset>>1), dest_cr +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[7]= 1;
1668             }
1669         }
1670     }
1671
1672     if(s->avctx->quantizer_noise_shaping){
1673         if(!skip_dct[0]) get_visual_weight(weight[0], ptr_y                 , wrap_y);
1674         if(!skip_dct[1]) get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
1675         if(!skip_dct[2]) get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
1676         if(!skip_dct[3]) get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
1677         if(!skip_dct[4]) get_visual_weight(weight[4], ptr_cb                , wrap_c);
1678         if(!skip_dct[5]) get_visual_weight(weight[5], ptr_cr                , wrap_c);
1679         if(!s->chroma_y_shift){ /* 422 */
1680             if(!skip_dct[6]) get_visual_weight(weight[6], ptr_cb + (dct_offset>>1), wrap_c);
1681             if(!skip_dct[7]) get_visual_weight(weight[7], ptr_cr + (dct_offset>>1), wrap_c);
1682         }
1683         memcpy(orig[0], s->block[0], sizeof(DCTELEM)*64*mb_block_count);
1684     }
1685
1686     /* DCT & quantize */
1687     assert(s->out_format!=FMT_MJPEG || s->qscale==8);
1688     {
1689         for(i=0;i<mb_block_count;i++) {
1690             if(!skip_dct[i]){
1691                 int overflow;
1692                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
1693             // FIXME we could decide to change to quantizer instead of clipping
1694             // JS: I don't think that would be a good idea it could lower quality instead
1695             //     of improve it. Just INTRADC clipping deserves changes in quantizer
1696                 if (overflow) clip_coeffs(s, s->block[i], s->block_last_index[i]);
1697             }else
1698                 s->block_last_index[i]= -1;
1699         }
1700         if(s->avctx->quantizer_noise_shaping){
1701             for(i=0;i<mb_block_count;i++) {
1702                 if(!skip_dct[i]){
1703                     s->block_last_index[i] = dct_quantize_refine(s, s->block[i], weight[i], orig[i], i, s->qscale);
1704                 }
1705             }
1706         }
1707
1708         if(s->luma_elim_threshold && !s->mb_intra)
1709             for(i=0; i<4; i++)
1710                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
1711         if(s->chroma_elim_threshold && !s->mb_intra)
1712             for(i=4; i<mb_block_count; i++)
1713                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
1714
1715         if(s->flags & CODEC_FLAG_CBP_RD){
1716             for(i=0;i<mb_block_count;i++) {
1717                 if(s->block_last_index[i] == -1)
1718                     s->coded_score[i]= INT_MAX/256;
1719             }
1720         }
1721     }
1722
1723     if((s->flags&CODEC_FLAG_GRAY) && s->mb_intra){
1724         s->block_last_index[4]=
1725         s->block_last_index[5]= 0;
1726         s->block[4][0]=
1727         s->block[5][0]= (1024 + s->c_dc_scale/2)/ s->c_dc_scale;
1728     }
1729
1730     //non c quantize code returns incorrect block_last_index FIXME
1731     if(s->alternate_scan && s->dct_quantize != dct_quantize_c){
1732         for(i=0; i<mb_block_count; i++){
1733             int j;
1734             if(s->block_last_index[i]>0){
1735                 for(j=63; j>0; j--){
1736                     if(s->block[i][ s->intra_scantable.permutated[j] ]) break;
1737                 }
1738                 s->block_last_index[i]= j;
1739             }
1740         }
1741     }
1742
1743     /* huffman encode */
1744     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
1745     case CODEC_ID_MPEG1VIDEO:
1746     case CODEC_ID_MPEG2VIDEO:
1747         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
1748             mpeg1_encode_mb(s, s->block, motion_x, motion_y);
1749         break;
1750     case CODEC_ID_MPEG4:
1751         if (CONFIG_MPEG4_ENCODER)
1752             mpeg4_encode_mb(s, s->block, motion_x, motion_y);
1753         break;
1754     case CODEC_ID_MSMPEG4V2:
1755     case CODEC_ID_MSMPEG4V3:
1756     case CODEC_ID_WMV1:
1757         if (CONFIG_MSMPEG4_ENCODER)
1758             msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
1759         break;
1760     case CODEC_ID_WMV2:
1761         if (CONFIG_WMV2_ENCODER)
1762             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
1763         break;
1764     case CODEC_ID_H261:
1765         if (CONFIG_H261_ENCODER)
1766             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
1767         break;
1768     case CODEC_ID_H263:
1769     case CODEC_ID_H263P:
1770     case CODEC_ID_FLV1:
1771     case CODEC_ID_RV10:
1772     case CODEC_ID_RV20:
1773         if (CONFIG_H263_ENCODER)
1774             h263_encode_mb(s, s->block, motion_x, motion_y);
1775         break;
1776     case CODEC_ID_MJPEG:
1777         if (CONFIG_MJPEG_ENCODER)
1778             ff_mjpeg_encode_mb(s, s->block);
1779         break;
1780     default:
1781         assert(0);
1782     }
1783 }
1784
1785 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
1786 {
1787     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
1788     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
1789 }
1790
1791 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
1792     int i;
1793
1794     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
1795
1796     /* mpeg1 */
1797     d->mb_skip_run= s->mb_skip_run;
1798     for(i=0; i<3; i++)
1799         d->last_dc[i]= s->last_dc[i];
1800
1801     /* statistics */
1802     d->mv_bits= s->mv_bits;
1803     d->i_tex_bits= s->i_tex_bits;
1804     d->p_tex_bits= s->p_tex_bits;
1805     d->i_count= s->i_count;
1806     d->f_count= s->f_count;
1807     d->b_count= s->b_count;
1808     d->skip_count= s->skip_count;
1809     d->misc_bits= s->misc_bits;
1810     d->last_bits= 0;
1811
1812     d->mb_skipped= 0;
1813     d->qscale= s->qscale;
1814     d->dquant= s->dquant;
1815
1816     d->esc3_level_length= s->esc3_level_length;
1817 }
1818
1819 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
1820     int i;
1821
1822     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
1823     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
1824
1825     /* mpeg1 */
1826     d->mb_skip_run= s->mb_skip_run;
1827     for(i=0; i<3; i++)
1828         d->last_dc[i]= s->last_dc[i];
1829
1830     /* statistics */
1831     d->mv_bits= s->mv_bits;
1832     d->i_tex_bits= s->i_tex_bits;
1833     d->p_tex_bits= s->p_tex_bits;
1834     d->i_count= s->i_count;
1835     d->f_count= s->f_count;
1836     d->b_count= s->b_count;
1837     d->skip_count= s->skip_count;
1838     d->misc_bits= s->misc_bits;
1839
1840     d->mb_intra= s->mb_intra;
1841     d->mb_skipped= s->mb_skipped;
1842     d->mv_type= s->mv_type;
1843     d->mv_dir= s->mv_dir;
1844     d->pb= s->pb;
1845     if(s->data_partitioning){
1846         d->pb2= s->pb2;
1847         d->tex_pb= s->tex_pb;
1848     }
1849     d->block= s->block;
1850     for(i=0; i<8; i++)
1851         d->block_last_index[i]= s->block_last_index[i];
1852     d->interlaced_dct= s->interlaced_dct;
1853     d->qscale= s->qscale;
1854
1855     d->esc3_level_length= s->esc3_level_length;
1856 }
1857
1858 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
1859                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
1860                            int *dmin, int *next_block, int motion_x, int motion_y)
1861 {
1862     int score;
1863     uint8_t *dest_backup[3];
1864
1865     copy_context_before_encode(s, backup, type);
1866
1867     s->block= s->blocks[*next_block];
1868     s->pb= pb[*next_block];
1869     if(s->data_partitioning){
1870         s->pb2   = pb2   [*next_block];
1871         s->tex_pb= tex_pb[*next_block];
1872     }
1873
1874     if(*next_block){
1875         memcpy(dest_backup, s->dest, sizeof(s->dest));
1876         s->dest[0] = s->rd_scratchpad;
1877         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
1878         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
1879         assert(s->linesize >= 32); //FIXME
1880     }
1881
1882     encode_mb(s, motion_x, motion_y);
1883
1884     score= put_bits_count(&s->pb);
1885     if(s->data_partitioning){
1886         score+= put_bits_count(&s->pb2);
1887         score+= put_bits_count(&s->tex_pb);
1888     }
1889
1890     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
1891         MPV_decode_mb(s, s->block);
1892
1893         score *= s->lambda2;
1894         score += sse_mb(s) << FF_LAMBDA_SHIFT;
1895     }
1896
1897     if(*next_block){
1898         memcpy(s->dest, dest_backup, sizeof(s->dest));
1899     }
1900
1901     if(score<*dmin){
1902         *dmin= score;
1903         *next_block^=1;
1904
1905         copy_context_after_encode(best, s, type);
1906     }
1907 }
1908
1909 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
1910     uint32_t *sq = ff_squareTbl + 256;
1911     int acc=0;
1912     int x,y;
1913
1914     if(w==16 && h==16)
1915         return s->dsp.sse[0](NULL, src1, src2, stride, 16);
1916     else if(w==8 && h==8)
1917         return s->dsp.sse[1](NULL, src1, src2, stride, 8);
1918
1919     for(y=0; y<h; y++){
1920         for(x=0; x<w; x++){
1921             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
1922         }
1923     }
1924
1925     assert(acc>=0);
1926
1927     return acc;
1928 }
1929
1930 static int sse_mb(MpegEncContext *s){
1931     int w= 16;
1932     int h= 16;
1933
1934     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
1935     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
1936
1937     if(w==16 && h==16)
1938       if(s->avctx->mb_cmp == FF_CMP_NSSE){
1939         return  s->dsp.nsse[0](s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
1940                +s->dsp.nsse[1](s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
1941                +s->dsp.nsse[1](s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
1942       }else{
1943         return  s->dsp.sse[0](NULL, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
1944                +s->dsp.sse[1](NULL, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
1945                +s->dsp.sse[1](NULL, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
1946       }
1947     else
1948         return  sse(s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
1949                +sse(s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
1950                +sse(s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
1951 }
1952
1953 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
1954     MpegEncContext *s= *(void**)arg;
1955
1956
1957     s->me.pre_pass=1;
1958     s->me.dia_size= s->avctx->pre_dia_size;
1959     s->first_slice_line=1;
1960     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
1961         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
1962             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
1963         }
1964         s->first_slice_line=0;
1965     }
1966
1967     s->me.pre_pass=0;
1968
1969     return 0;
1970 }
1971
1972 static int estimate_motion_thread(AVCodecContext *c, void *arg){
1973     MpegEncContext *s= *(void**)arg;
1974
1975     ff_check_alignment();
1976
1977     s->me.dia_size= s->avctx->dia_size;
1978     s->first_slice_line=1;
1979     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
1980         s->mb_x=0; //for block init below
1981         ff_init_block_index(s);
1982         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
1983             s->block_index[0]+=2;
1984             s->block_index[1]+=2;
1985             s->block_index[2]+=2;
1986             s->block_index[3]+=2;
1987
1988             /* compute motion vector & mb_type and store in context */
1989             if(s->pict_type==AV_PICTURE_TYPE_B)
1990                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
1991             else
1992                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
1993         }
1994         s->first_slice_line=0;
1995     }
1996     return 0;
1997 }
1998
1999 static int mb_var_thread(AVCodecContext *c, void *arg){
2000     MpegEncContext *s= *(void**)arg;
2001     int mb_x, mb_y;
2002
2003     ff_check_alignment();
2004
2005     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2006         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2007             int xx = mb_x * 16;
2008             int yy = mb_y * 16;
2009             uint8_t *pix = s->new_picture.f.data[0] + (yy * s->linesize) + xx;
2010             int varc;
2011             int sum = s->dsp.pix_sum(pix, s->linesize);
2012
2013             varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)(sum*sum))>>8) + 500 + 128)>>8;
2014
2015             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2016             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2017             s->me.mb_var_sum_temp    += varc;
2018         }
2019     }
2020     return 0;
2021 }
2022
2023 static void write_slice_end(MpegEncContext *s){
2024     if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4){
2025         if(s->partitioned_frame){
2026             ff_mpeg4_merge_partitions(s);
2027         }
2028
2029         ff_mpeg4_stuffing(&s->pb);
2030     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2031         ff_mjpeg_encode_stuffing(&s->pb);
2032     }
2033
2034     align_put_bits(&s->pb);
2035     flush_put_bits(&s->pb);
2036
2037     if((s->flags&CODEC_FLAG_PASS1) && !s->partitioned_frame)
2038         s->misc_bits+= get_bits_diff(s);
2039 }
2040
2041 static int encode_thread(AVCodecContext *c, void *arg){
2042     MpegEncContext *s= *(void**)arg;
2043     int mb_x, mb_y, pdif = 0;
2044     int chr_h= 16>>s->chroma_y_shift;
2045     int i, j;
2046     MpegEncContext best_s, backup_s;
2047     uint8_t bit_buf[2][MAX_MB_BYTES];
2048     uint8_t bit_buf2[2][MAX_MB_BYTES];
2049     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2050     PutBitContext pb[2], pb2[2], tex_pb[2];
2051 //printf("%d->%d\n", s->resync_mb_y, s->end_mb_y);
2052
2053     ff_check_alignment();
2054
2055     for(i=0; i<2; i++){
2056         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2057         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2058         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2059     }
2060
2061     s->last_bits= put_bits_count(&s->pb);
2062     s->mv_bits=0;
2063     s->misc_bits=0;
2064     s->i_tex_bits=0;
2065     s->p_tex_bits=0;
2066     s->i_count=0;
2067     s->f_count=0;
2068     s->b_count=0;
2069     s->skip_count=0;
2070
2071     for(i=0; i<3; i++){
2072         /* init last dc values */
2073         /* note: quant matrix value (8) is implied here */
2074         s->last_dc[i] = 128 << s->intra_dc_precision;
2075
2076         s->current_picture.f.error[i] = 0;
2077     }
2078     s->mb_skip_run = 0;
2079     memset(s->last_mv, 0, sizeof(s->last_mv));
2080
2081     s->last_mv_dir = 0;
2082
2083     switch(s->codec_id){
2084     case CODEC_ID_H263:
2085     case CODEC_ID_H263P:
2086     case CODEC_ID_FLV1:
2087         if (CONFIG_H263_ENCODER)
2088             s->gob_index = ff_h263_get_gob_height(s);
2089         break;
2090     case CODEC_ID_MPEG4:
2091         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2092             ff_mpeg4_init_partitions(s);
2093         break;
2094     }
2095
2096     s->resync_mb_x=0;
2097     s->resync_mb_y=0;
2098     s->first_slice_line = 1;
2099     s->ptr_lastgob = s->pb.buf;
2100     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2101 //    printf("row %d at %X\n", s->mb_y, (int)s);
2102         s->mb_x=0;
2103         s->mb_y= mb_y;
2104
2105         ff_set_qscale(s, s->qscale);
2106         ff_init_block_index(s);
2107
2108         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2109             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2110             int mb_type= s->mb_type[xy];
2111 //            int d;
2112             int dmin= INT_MAX;
2113             int dir;
2114
2115             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2116                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2117                 return -1;
2118             }
2119             if(s->data_partitioning){
2120                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2121                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2122                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2123                     return -1;
2124                 }
2125             }
2126
2127             s->mb_x = mb_x;
2128             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2129             ff_update_block_index(s);
2130
2131             if(CONFIG_H261_ENCODER && s->codec_id == CODEC_ID_H261){
2132                 ff_h261_reorder_mb_index(s);
2133                 xy= s->mb_y*s->mb_stride + s->mb_x;
2134                 mb_type= s->mb_type[xy];
2135             }
2136
2137             /* write gob / video packet header  */
2138             if(s->rtp_mode){
2139                 int current_packet_size, is_gob_start;
2140
2141                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2142
2143                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2144
2145                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2146
2147                 switch(s->codec_id){
2148                 case CODEC_ID_H263:
2149                 case CODEC_ID_H263P:
2150                     if(!s->h263_slice_structured)
2151                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2152                     break;
2153                 case CODEC_ID_MPEG2VIDEO:
2154                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2155                 case CODEC_ID_MPEG1VIDEO:
2156                     if(s->mb_skip_run) is_gob_start=0;
2157                     break;
2158                 }
2159
2160                 if(is_gob_start){
2161                     if(s->start_mb_y != mb_y || mb_x!=0){
2162                         write_slice_end(s);
2163
2164                         if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4 && s->partitioned_frame){
2165                             ff_mpeg4_init_partitions(s);
2166                         }
2167                     }
2168
2169                     assert((put_bits_count(&s->pb)&7) == 0);
2170                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2171
2172                     if(s->avctx->error_rate && s->resync_mb_x + s->resync_mb_y > 0){
2173                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2174                         int d= 100 / s->avctx->error_rate;
2175                         if(r % d == 0){
2176                             current_packet_size=0;
2177                             s->pb.buf_ptr= s->ptr_lastgob;
2178                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2179                         }
2180                     }
2181
2182                     if (s->avctx->rtp_callback){
2183                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2184                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2185                     }
2186
2187                     switch(s->codec_id){
2188                     case CODEC_ID_MPEG4:
2189                         if (CONFIG_MPEG4_ENCODER) {
2190                             ff_mpeg4_encode_video_packet_header(s);
2191                             ff_mpeg4_clean_buffers(s);
2192                         }
2193                     break;
2194                     case CODEC_ID_MPEG1VIDEO:
2195                     case CODEC_ID_MPEG2VIDEO:
2196                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2197                             ff_mpeg1_encode_slice_header(s);
2198                             ff_mpeg1_clean_buffers(s);
2199                         }
2200                     break;
2201                     case CODEC_ID_H263:
2202                     case CODEC_ID_H263P:
2203                         if (CONFIG_H263_ENCODER)
2204                             h263_encode_gob_header(s, mb_y);
2205                     break;
2206                     }
2207
2208                     if(s->flags&CODEC_FLAG_PASS1){
2209                         int bits= put_bits_count(&s->pb);
2210                         s->misc_bits+= bits - s->last_bits;
2211                         s->last_bits= bits;
2212                     }
2213
2214                     s->ptr_lastgob += current_packet_size;
2215                     s->first_slice_line=1;
2216                     s->resync_mb_x=mb_x;
2217                     s->resync_mb_y=mb_y;
2218                 }
2219             }
2220
2221             if(  (s->resync_mb_x   == s->mb_x)
2222                && s->resync_mb_y+1 == s->mb_y){
2223                 s->first_slice_line=0;
2224             }
2225
2226             s->mb_skipped=0;
2227             s->dquant=0; //only for QP_RD
2228
2229             if(mb_type & (mb_type-1) || (s->flags & CODEC_FLAG_QP_RD)){ // more than 1 MB type possible or CODEC_FLAG_QP_RD
2230                 int next_block=0;
2231                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2232
2233                 copy_context_before_encode(&backup_s, s, -1);
2234                 backup_s.pb= s->pb;
2235                 best_s.data_partitioning= s->data_partitioning;
2236                 best_s.partitioned_frame= s->partitioned_frame;
2237                 if(s->data_partitioning){
2238                     backup_s.pb2= s->pb2;
2239                     backup_s.tex_pb= s->tex_pb;
2240                 }
2241
2242                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2243                     s->mv_dir = MV_DIR_FORWARD;
2244                     s->mv_type = MV_TYPE_16X16;
2245                     s->mb_intra= 0;
2246                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2247                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2248                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2249                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2250                 }
2251                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2252                     s->mv_dir = MV_DIR_FORWARD;
2253                     s->mv_type = MV_TYPE_FIELD;
2254                     s->mb_intra= 0;
2255                     for(i=0; i<2; i++){
2256                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2257                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2258                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2259                     }
2260                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2261                                  &dmin, &next_block, 0, 0);
2262                 }
2263                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2264                     s->mv_dir = MV_DIR_FORWARD;
2265                     s->mv_type = MV_TYPE_16X16;
2266                     s->mb_intra= 0;
2267                     s->mv[0][0][0] = 0;
2268                     s->mv[0][0][1] = 0;
2269                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2270                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2271                 }
2272                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2273                     s->mv_dir = MV_DIR_FORWARD;
2274                     s->mv_type = MV_TYPE_8X8;
2275                     s->mb_intra= 0;
2276                     for(i=0; i<4; i++){
2277                         s->mv[0][i][0] = s->current_picture.f.motion_val[0][s->block_index[i]][0];
2278                         s->mv[0][i][1] = s->current_picture.f.motion_val[0][s->block_index[i]][1];
2279                     }
2280                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2281                                  &dmin, &next_block, 0, 0);
2282                 }
2283                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2284                     s->mv_dir = MV_DIR_FORWARD;
2285                     s->mv_type = MV_TYPE_16X16;
2286                     s->mb_intra= 0;
2287                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2288                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2289                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2290                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2291                 }
2292                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2293                     s->mv_dir = MV_DIR_BACKWARD;
2294                     s->mv_type = MV_TYPE_16X16;
2295                     s->mb_intra= 0;
2296                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2297                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2298                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2299                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2300                 }
2301                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2302                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2303                     s->mv_type = MV_TYPE_16X16;
2304                     s->mb_intra= 0;
2305                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2306                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2307                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2308                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2309                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2310                                  &dmin, &next_block, 0, 0);
2311                 }
2312                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2313                     s->mv_dir = MV_DIR_FORWARD;
2314                     s->mv_type = MV_TYPE_FIELD;
2315                     s->mb_intra= 0;
2316                     for(i=0; i<2; i++){
2317                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2318                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2319                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2320                     }
2321                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2322                                  &dmin, &next_block, 0, 0);
2323                 }
2324                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2325                     s->mv_dir = MV_DIR_BACKWARD;
2326                     s->mv_type = MV_TYPE_FIELD;
2327                     s->mb_intra= 0;
2328                     for(i=0; i<2; i++){
2329                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2330                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2331                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2332                     }
2333                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2334                                  &dmin, &next_block, 0, 0);
2335                 }
2336                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2337                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2338                     s->mv_type = MV_TYPE_FIELD;
2339                     s->mb_intra= 0;
2340                     for(dir=0; dir<2; dir++){
2341                         for(i=0; i<2; i++){
2342                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2343                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2344                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2345                         }
2346                     }
2347                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2348                                  &dmin, &next_block, 0, 0);
2349                 }
2350                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2351                     s->mv_dir = 0;
2352                     s->mv_type = MV_TYPE_16X16;
2353                     s->mb_intra= 1;
2354                     s->mv[0][0][0] = 0;
2355                     s->mv[0][0][1] = 0;
2356                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2357                                  &dmin, &next_block, 0, 0);
2358                     if(s->h263_pred || s->h263_aic){
2359                         if(best_s.mb_intra)
2360                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2361                         else
2362                             ff_clean_intra_table_entries(s); //old mode?
2363                     }
2364                 }
2365
2366                 if((s->flags & CODEC_FLAG_QP_RD) && dmin < INT_MAX){
2367                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2368                         const int last_qp= backup_s.qscale;
2369                         int qpi, qp, dc[6];
2370                         DCTELEM ac[6][16];
2371                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2372                         static const int dquant_tab[4]={-1,1,-2,2};
2373
2374                         assert(backup_s.dquant == 0);
2375
2376                         //FIXME intra
2377                         s->mv_dir= best_s.mv_dir;
2378                         s->mv_type = MV_TYPE_16X16;
2379                         s->mb_intra= best_s.mb_intra;
2380                         s->mv[0][0][0] = best_s.mv[0][0][0];
2381                         s->mv[0][0][1] = best_s.mv[0][0][1];
2382                         s->mv[1][0][0] = best_s.mv[1][0][0];
2383                         s->mv[1][0][1] = best_s.mv[1][0][1];
2384
2385                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
2386                         for(; qpi<4; qpi++){
2387                             int dquant= dquant_tab[qpi];
2388                             qp= last_qp + dquant;
2389                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2390                                 continue;
2391                             backup_s.dquant= dquant;
2392                             if(s->mb_intra && s->dc_val[0]){
2393                                 for(i=0; i<6; i++){
2394                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2395                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(DCTELEM)*16);
2396                                 }
2397                             }
2398
2399                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2400                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2401                             if(best_s.qscale != qp){
2402                                 if(s->mb_intra && s->dc_val[0]){
2403                                     for(i=0; i<6; i++){
2404                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2405                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(DCTELEM)*16);
2406                                     }
2407                                 }
2408                             }
2409                         }
2410                     }
2411                 }
2412                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2413                     int mx= s->b_direct_mv_table[xy][0];
2414                     int my= s->b_direct_mv_table[xy][1];
2415
2416                     backup_s.dquant = 0;
2417                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2418                     s->mb_intra= 0;
2419                     ff_mpeg4_set_direct_mv(s, mx, my);
2420                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2421                                  &dmin, &next_block, mx, my);
2422                 }
2423                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2424                     backup_s.dquant = 0;
2425                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2426                     s->mb_intra= 0;
2427                     ff_mpeg4_set_direct_mv(s, 0, 0);
2428                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2429                                  &dmin, &next_block, 0, 0);
2430                 }
2431                 if(!best_s.mb_intra && s->flags2&CODEC_FLAG2_SKIP_RD){
2432                     int coded=0;
2433                     for(i=0; i<6; i++)
2434                         coded |= s->block_last_index[i];
2435                     if(coded){
2436                         int mx,my;
2437                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2438                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
2439                             mx=my=0; //FIXME find the one we actually used
2440                             ff_mpeg4_set_direct_mv(s, mx, my);
2441                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2442                             mx= s->mv[1][0][0];
2443                             my= s->mv[1][0][1];
2444                         }else{
2445                             mx= s->mv[0][0][0];
2446                             my= s->mv[0][0][1];
2447                         }
2448
2449                         s->mv_dir= best_s.mv_dir;
2450                         s->mv_type = best_s.mv_type;
2451                         s->mb_intra= 0;
2452 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2453                         s->mv[0][0][1] = best_s.mv[0][0][1];
2454                         s->mv[1][0][0] = best_s.mv[1][0][0];
2455                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2456                         backup_s.dquant= 0;
2457                         s->skipdct=1;
2458                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2459                                         &dmin, &next_block, mx, my);
2460                         s->skipdct=0;
2461                     }
2462                 }
2463
2464                 s->current_picture.f.qscale_table[xy] = best_s.qscale;
2465
2466                 copy_context_after_encode(s, &best_s, -1);
2467
2468                 pb_bits_count= put_bits_count(&s->pb);
2469                 flush_put_bits(&s->pb);
2470                 ff_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2471                 s->pb= backup_s.pb;
2472
2473                 if(s->data_partitioning){
2474                     pb2_bits_count= put_bits_count(&s->pb2);
2475                     flush_put_bits(&s->pb2);
2476                     ff_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2477                     s->pb2= backup_s.pb2;
2478
2479                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2480                     flush_put_bits(&s->tex_pb);
2481                     ff_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2482                     s->tex_pb= backup_s.tex_pb;
2483                 }
2484                 s->last_bits= put_bits_count(&s->pb);
2485
2486                 if (CONFIG_H263_ENCODER &&
2487                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2488                     ff_h263_update_motion_val(s);
2489
2490                 if(next_block==0){ //FIXME 16 vs linesize16
2491                     s->dsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
2492                     s->dsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2493                     s->dsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2494                 }
2495
2496                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
2497                     MPV_decode_mb(s, s->block);
2498             } else {
2499                 int motion_x = 0, motion_y = 0;
2500                 s->mv_type=MV_TYPE_16X16;
2501                 // only one MB-Type possible
2502
2503                 switch(mb_type){
2504                 case CANDIDATE_MB_TYPE_INTRA:
2505                     s->mv_dir = 0;
2506                     s->mb_intra= 1;
2507                     motion_x= s->mv[0][0][0] = 0;
2508                     motion_y= s->mv[0][0][1] = 0;
2509                     break;
2510                 case CANDIDATE_MB_TYPE_INTER:
2511                     s->mv_dir = MV_DIR_FORWARD;
2512                     s->mb_intra= 0;
2513                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
2514                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
2515                     break;
2516                 case CANDIDATE_MB_TYPE_INTER_I:
2517                     s->mv_dir = MV_DIR_FORWARD;
2518                     s->mv_type = MV_TYPE_FIELD;
2519                     s->mb_intra= 0;
2520                     for(i=0; i<2; i++){
2521                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2522                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2523                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2524                     }
2525                     break;
2526                 case CANDIDATE_MB_TYPE_INTER4V:
2527                     s->mv_dir = MV_DIR_FORWARD;
2528                     s->mv_type = MV_TYPE_8X8;
2529                     s->mb_intra= 0;
2530                     for(i=0; i<4; i++){
2531                         s->mv[0][i][0] = s->current_picture.f.motion_val[0][s->block_index[i]][0];
2532                         s->mv[0][i][1] = s->current_picture.f.motion_val[0][s->block_index[i]][1];
2533                     }
2534                     break;
2535                 case CANDIDATE_MB_TYPE_DIRECT:
2536                     if (CONFIG_MPEG4_ENCODER) {
2537                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2538                         s->mb_intra= 0;
2539                         motion_x=s->b_direct_mv_table[xy][0];
2540                         motion_y=s->b_direct_mv_table[xy][1];
2541                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
2542                     }
2543                     break;
2544                 case CANDIDATE_MB_TYPE_DIRECT0:
2545                     if (CONFIG_MPEG4_ENCODER) {
2546                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2547                         s->mb_intra= 0;
2548                         ff_mpeg4_set_direct_mv(s, 0, 0);
2549                     }
2550                     break;
2551                 case CANDIDATE_MB_TYPE_BIDIR:
2552                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2553                     s->mb_intra= 0;
2554                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2555                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2556                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2557                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2558                     break;
2559                 case CANDIDATE_MB_TYPE_BACKWARD:
2560                     s->mv_dir = MV_DIR_BACKWARD;
2561                     s->mb_intra= 0;
2562                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2563                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2564                     break;
2565                 case CANDIDATE_MB_TYPE_FORWARD:
2566                     s->mv_dir = MV_DIR_FORWARD;
2567                     s->mb_intra= 0;
2568                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2569                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2570 //                    printf(" %d %d ", motion_x, motion_y);
2571                     break;
2572                 case CANDIDATE_MB_TYPE_FORWARD_I:
2573                     s->mv_dir = MV_DIR_FORWARD;
2574                     s->mv_type = MV_TYPE_FIELD;
2575                     s->mb_intra= 0;
2576                     for(i=0; i<2; i++){
2577                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2578                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2579                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2580                     }
2581                     break;
2582                 case CANDIDATE_MB_TYPE_BACKWARD_I:
2583                     s->mv_dir = MV_DIR_BACKWARD;
2584                     s->mv_type = MV_TYPE_FIELD;
2585                     s->mb_intra= 0;
2586                     for(i=0; i<2; i++){
2587                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2588                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2589                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2590                     }
2591                     break;
2592                 case CANDIDATE_MB_TYPE_BIDIR_I:
2593                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2594                     s->mv_type = MV_TYPE_FIELD;
2595                     s->mb_intra= 0;
2596                     for(dir=0; dir<2; dir++){
2597                         for(i=0; i<2; i++){
2598                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2599                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2600                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2601                         }
2602                     }
2603                     break;
2604                 default:
2605                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
2606                 }
2607
2608                 encode_mb(s, motion_x, motion_y);
2609
2610                 // RAL: Update last macroblock type
2611                 s->last_mv_dir = s->mv_dir;
2612
2613                 if (CONFIG_H263_ENCODER &&
2614                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2615                     ff_h263_update_motion_val(s);
2616
2617                 MPV_decode_mb(s, s->block);
2618             }
2619
2620             /* clean the MV table in IPS frames for direct mode in B frames */
2621             if(s->mb_intra /* && I,P,S_TYPE */){
2622                 s->p_mv_table[xy][0]=0;
2623                 s->p_mv_table[xy][1]=0;
2624             }
2625
2626             if(s->flags&CODEC_FLAG_PSNR){
2627                 int w= 16;
2628                 int h= 16;
2629
2630                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2631                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2632
2633                 s->current_picture.f.error[0] += sse(
2634                     s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
2635                     s->dest[0], w, h, s->linesize);
2636                 s->current_picture.f.error[1] += sse(
2637                     s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2638                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2639                 s->current_picture.f.error[2] += sse(
2640                     s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2641                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2642             }
2643             if(s->loop_filter){
2644                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
2645                     ff_h263_loop_filter(s);
2646             }
2647 //printf("MB %d %d bits\n", s->mb_x+s->mb_y*s->mb_stride, put_bits_count(&s->pb));
2648         }
2649     }
2650
2651     //not beautiful here but we must write it before flushing so it has to be here
2652     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
2653         msmpeg4_encode_ext_header(s);
2654
2655     write_slice_end(s);
2656
2657     /* Send the last GOB if RTP */
2658     if (s->avctx->rtp_callback) {
2659         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
2660         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
2661         /* Call the RTP callback to send the last GOB */
2662         emms_c();
2663         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
2664     }
2665
2666     return 0;
2667 }
2668
2669 #define MERGE(field) dst->field += src->field; src->field=0
2670 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
2671     MERGE(me.scene_change_score);
2672     MERGE(me.mc_mb_var_sum_temp);
2673     MERGE(me.mb_var_sum_temp);
2674 }
2675
2676 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
2677     int i;
2678
2679     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
2680     MERGE(dct_count[1]);
2681     MERGE(mv_bits);
2682     MERGE(i_tex_bits);
2683     MERGE(p_tex_bits);
2684     MERGE(i_count);
2685     MERGE(f_count);
2686     MERGE(b_count);
2687     MERGE(skip_count);
2688     MERGE(misc_bits);
2689     MERGE(error_count);
2690     MERGE(padding_bug_score);
2691     MERGE(current_picture.f.error[0]);
2692     MERGE(current_picture.f.error[1]);
2693     MERGE(current_picture.f.error[2]);
2694
2695     if(dst->avctx->noise_reduction){
2696         for(i=0; i<64; i++){
2697             MERGE(dct_error_sum[0][i]);
2698             MERGE(dct_error_sum[1][i]);
2699         }
2700     }
2701
2702     assert(put_bits_count(&src->pb) % 8 ==0);
2703     assert(put_bits_count(&dst->pb) % 8 ==0);
2704     ff_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
2705     flush_put_bits(&dst->pb);
2706 }
2707
2708 static int estimate_qp(MpegEncContext *s, int dry_run){
2709     if (s->next_lambda){
2710         s->current_picture_ptr->f.quality =
2711         s->current_picture.f.quality = s->next_lambda;
2712         if(!dry_run) s->next_lambda= 0;
2713     } else if (!s->fixed_qscale) {
2714         s->current_picture_ptr->f.quality =
2715         s->current_picture.f.quality = ff_rate_estimate_qscale(s, dry_run);
2716         if (s->current_picture.f.quality < 0)
2717             return -1;
2718     }
2719
2720     if(s->adaptive_quant){
2721         switch(s->codec_id){
2722         case CODEC_ID_MPEG4:
2723             if (CONFIG_MPEG4_ENCODER)
2724                 ff_clean_mpeg4_qscales(s);
2725             break;
2726         case CODEC_ID_H263:
2727         case CODEC_ID_H263P:
2728         case CODEC_ID_FLV1:
2729             if (CONFIG_H263_ENCODER)
2730                 ff_clean_h263_qscales(s);
2731             break;
2732         default:
2733             ff_init_qscale_tab(s);
2734         }
2735
2736         s->lambda= s->lambda_table[0];
2737         //FIXME broken
2738     }else
2739         s->lambda = s->current_picture.f.quality;
2740 //printf("%d %d\n", s->avctx->global_quality, s->current_picture.quality);
2741     update_qscale(s);
2742     return 0;
2743 }
2744
2745 /* must be called before writing the header */
2746 static void set_frame_distances(MpegEncContext * s){
2747     assert(s->current_picture_ptr->pts != AV_NOPTS_VALUE);
2748     s->time = s->current_picture_ptr->f.pts * s->avctx->time_base.num;
2749
2750     if(s->pict_type==AV_PICTURE_TYPE_B){
2751         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
2752         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
2753     }else{
2754         s->pp_time= s->time - s->last_non_b_time;
2755         s->last_non_b_time= s->time;
2756         assert(s->picture_number==0 || s->pp_time > 0);
2757     }
2758 }
2759
2760 static int encode_picture(MpegEncContext *s, int picture_number)
2761 {
2762     int i;
2763     int bits;
2764     int context_count = s->avctx->thread_count;
2765
2766     s->picture_number = picture_number;
2767
2768     /* Reset the average MB variance */
2769     s->me.mb_var_sum_temp    =
2770     s->me.mc_mb_var_sum_temp = 0;
2771
2772     /* we need to initialize some time vars before we can encode b-frames */
2773     // RAL: Condition added for MPEG1VIDEO
2774     if (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
2775         set_frame_distances(s);
2776     if(CONFIG_MPEG4_ENCODER && s->codec_id == CODEC_ID_MPEG4)
2777         ff_set_mpeg4_time(s);
2778
2779     s->me.scene_change_score=0;
2780
2781 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
2782
2783     if(s->pict_type==AV_PICTURE_TYPE_I){
2784         if(s->msmpeg4_version >= 3) s->no_rounding=1;
2785         else                        s->no_rounding=0;
2786     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
2787         if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
2788             s->no_rounding ^= 1;
2789     }
2790
2791     if(s->flags & CODEC_FLAG_PASS2){
2792         if (estimate_qp(s,1) < 0)
2793             return -1;
2794         ff_get_2pass_fcode(s);
2795     }else if(!(s->flags & CODEC_FLAG_QSCALE)){
2796         if(s->pict_type==AV_PICTURE_TYPE_B)
2797             s->lambda= s->last_lambda_for[s->pict_type];
2798         else
2799             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
2800         update_qscale(s);
2801     }
2802
2803     s->mb_intra=0; //for the rate distortion & bit compare functions
2804     for(i=1; i<context_count; i++){
2805         ff_update_duplicate_context(s->thread_context[i], s);
2806     }
2807
2808     if(ff_init_me(s)<0)
2809         return -1;
2810
2811     /* Estimate motion for every MB */
2812     if(s->pict_type != AV_PICTURE_TYPE_I){
2813         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
2814         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
2815         if(s->pict_type != AV_PICTURE_TYPE_B && s->avctx->me_threshold==0){
2816             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
2817                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
2818             }
2819         }
2820
2821         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
2822     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
2823         /* I-Frame */
2824         for(i=0; i<s->mb_stride*s->mb_height; i++)
2825             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
2826
2827         if(!s->fixed_qscale){
2828             /* finding spatial complexity for I-frame rate control */
2829             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
2830         }
2831     }
2832     for(i=1; i<context_count; i++){
2833         merge_context_after_me(s, s->thread_context[i]);
2834     }
2835     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
2836     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
2837     emms_c();
2838
2839     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
2840         s->pict_type= AV_PICTURE_TYPE_I;
2841         for(i=0; i<s->mb_stride*s->mb_height; i++)
2842             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
2843 //printf("Scene change detected, encoding as I Frame %d %d\n", s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
2844     }
2845
2846     if(!s->umvplus){
2847         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
2848             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
2849
2850             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2851                 int a,b;
2852                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
2853                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
2854                 s->f_code= FFMAX3(s->f_code, a, b);
2855             }
2856
2857             ff_fix_long_p_mvs(s);
2858             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
2859             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2860                 int j;
2861                 for(i=0; i<2; i++){
2862                     for(j=0; j<2; j++)
2863                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
2864                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
2865                 }
2866             }
2867         }
2868
2869         if(s->pict_type==AV_PICTURE_TYPE_B){
2870             int a, b;
2871
2872             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
2873             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
2874             s->f_code = FFMAX(a, b);
2875
2876             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
2877             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
2878             s->b_code = FFMAX(a, b);
2879
2880             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
2881             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
2882             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
2883             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
2884             if(s->flags & CODEC_FLAG_INTERLACED_ME){
2885                 int dir, j;
2886                 for(dir=0; dir<2; dir++){
2887                     for(i=0; i<2; i++){
2888                         for(j=0; j<2; j++){
2889                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
2890                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
2891                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
2892                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
2893                         }
2894                     }
2895                 }
2896             }
2897         }
2898     }
2899
2900     if (estimate_qp(s, 0) < 0)
2901         return -1;
2902
2903     if(s->qscale < 3 && s->max_qcoeff<=128 && s->pict_type==AV_PICTURE_TYPE_I && !(s->flags & CODEC_FLAG_QSCALE))
2904         s->qscale= 3; //reduce clipping problems
2905
2906     if (s->out_format == FMT_MJPEG) {
2907         /* for mjpeg, we do include qscale in the matrix */
2908         for(i=1;i<64;i++){
2909             int j= s->dsp.idct_permutation[i];
2910
2911             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
2912         }
2913         s->y_dc_scale_table=
2914         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
2915         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
2916         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
2917                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
2918         s->qscale= 8;
2919     }
2920
2921     //FIXME var duplication
2922     s->current_picture_ptr->f.key_frame =
2923     s->current_picture.f.key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
2924     s->current_picture_ptr->f.pict_type =
2925     s->current_picture.f.pict_type = s->pict_type;
2926
2927     if (s->current_picture.f.key_frame)
2928         s->picture_in_gop_number=0;
2929
2930     s->last_bits= put_bits_count(&s->pb);
2931     switch(s->out_format) {
2932     case FMT_MJPEG:
2933         if (CONFIG_MJPEG_ENCODER)
2934             ff_mjpeg_encode_picture_header(s);
2935         break;
2936     case FMT_H261:
2937         if (CONFIG_H261_ENCODER)
2938             ff_h261_encode_picture_header(s, picture_number);
2939         break;
2940     case FMT_H263:
2941         if (CONFIG_WMV2_ENCODER && s->codec_id == CODEC_ID_WMV2)
2942             ff_wmv2_encode_picture_header(s, picture_number);
2943         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
2944             msmpeg4_encode_picture_header(s, picture_number);
2945         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
2946             mpeg4_encode_picture_header(s, picture_number);
2947         else if (CONFIG_RV10_ENCODER && s->codec_id == CODEC_ID_RV10)
2948             rv10_encode_picture_header(s, picture_number);
2949         else if (CONFIG_RV20_ENCODER && s->codec_id == CODEC_ID_RV20)
2950             rv20_encode_picture_header(s, picture_number);
2951         else if (CONFIG_FLV_ENCODER && s->codec_id == CODEC_ID_FLV1)
2952             ff_flv_encode_picture_header(s, picture_number);
2953         else if (CONFIG_H263_ENCODER)
2954             h263_encode_picture_header(s, picture_number);
2955         break;
2956     case FMT_MPEG1:
2957         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2958             mpeg1_encode_picture_header(s, picture_number);
2959         break;
2960     case FMT_H264:
2961         break;
2962     default:
2963         assert(0);
2964     }
2965     bits= put_bits_count(&s->pb);
2966     s->header_bits= bits - s->last_bits;
2967
2968     for(i=1; i<context_count; i++){
2969         update_duplicate_context_after_me(s->thread_context[i], s);
2970     }
2971     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
2972     for(i=1; i<context_count; i++){
2973         merge_context_after_encode(s, s->thread_context[i]);
2974     }
2975     emms_c();
2976     return 0;
2977 }
2978
2979 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block){
2980     const int intra= s->mb_intra;
2981     int i;
2982
2983     s->dct_count[intra]++;
2984
2985     for(i=0; i<64; i++){
2986         int level= block[i];
2987
2988         if(level){
2989             if(level>0){
2990                 s->dct_error_sum[intra][i] += level;
2991                 level -= s->dct_offset[intra][i];
2992                 if(level<0) level=0;
2993             }else{
2994                 s->dct_error_sum[intra][i] -= level;
2995                 level += s->dct_offset[intra][i];
2996                 if(level>0) level=0;
2997             }
2998             block[i]= level;
2999         }
3000     }
3001 }
3002
3003 static int dct_quantize_trellis_c(MpegEncContext *s,
3004                                   DCTELEM *block, int n,
3005                                   int qscale, int *overflow){
3006     const int *qmat;
3007     const uint8_t *scantable= s->intra_scantable.scantable;
3008     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3009     int max=0;
3010     unsigned int threshold1, threshold2;
3011     int bias=0;
3012     int run_tab[65];
3013     int level_tab[65];
3014     int score_tab[65];
3015     int survivor[65];
3016     int survivor_count;
3017     int last_run=0;
3018     int last_level=0;
3019     int last_score= 0;
3020     int last_i;
3021     int coeff[2][64];
3022     int coeff_count[64];
3023     int qmul, qadd, start_i, last_non_zero, i, dc;
3024     const int esc_length= s->ac_esc_length;
3025     uint8_t * length;
3026     uint8_t * last_length;
3027     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3028
3029     s->dsp.fdct (block);
3030
3031     if(s->dct_error_sum)
3032         s->denoise_dct(s, block);
3033     qmul= qscale*16;
3034     qadd= ((qscale-1)|1)*8;
3035
3036     if (s->mb_intra) {
3037         int q;
3038         if (!s->h263_aic) {
3039             if (n < 4)
3040                 q = s->y_dc_scale;
3041             else
3042                 q = s->c_dc_scale;
3043             q = q << 3;
3044         } else{
3045             /* For AIC we skip quant/dequant of INTRADC */
3046             q = 1 << 3;
3047             qadd=0;
3048         }
3049
3050         /* note: block[0] is assumed to be positive */
3051         block[0] = (block[0] + (q >> 1)) / q;
3052         start_i = 1;
3053         last_non_zero = 0;
3054         qmat = s->q_intra_matrix[qscale];
3055         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3056             bias= 1<<(QMAT_SHIFT-1);
3057         length     = s->intra_ac_vlc_length;
3058         last_length= s->intra_ac_vlc_last_length;
3059     } else {
3060         start_i = 0;
3061         last_non_zero = -1;
3062         qmat = s->q_inter_matrix[qscale];
3063         length     = s->inter_ac_vlc_length;
3064         last_length= s->inter_ac_vlc_last_length;
3065     }
3066     last_i= start_i;
3067
3068     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3069     threshold2= (threshold1<<1);
3070
3071     for(i=63; i>=start_i; i--) {
3072         const int j = scantable[i];
3073         int level = block[j] * qmat[j];
3074
3075         if(((unsigned)(level+threshold1))>threshold2){
3076             last_non_zero = i;
3077             break;
3078         }
3079     }
3080
3081     for(i=start_i; i<=last_non_zero; i++) {
3082         const int j = scantable[i];
3083         int level = block[j] * qmat[j];
3084
3085 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3086 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3087         if(((unsigned)(level+threshold1))>threshold2){
3088             if(level>0){
3089                 level= (bias + level)>>QMAT_SHIFT;
3090                 coeff[0][i]= level;
3091                 coeff[1][i]= level-1;
3092 //                coeff[2][k]= level-2;
3093             }else{
3094                 level= (bias - level)>>QMAT_SHIFT;
3095                 coeff[0][i]= -level;
3096                 coeff[1][i]= -level+1;
3097 //                coeff[2][k]= -level+2;
3098             }
3099             coeff_count[i]= FFMIN(level, 2);
3100             assert(coeff_count[i]);
3101             max |=level;
3102         }else{
3103             coeff[0][i]= (level>>31)|1;
3104             coeff_count[i]= 1;
3105         }
3106     }
3107
3108     *overflow= s->max_qcoeff < max; //overflow might have happened
3109
3110     if(last_non_zero < start_i){
3111         memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3112         return last_non_zero;
3113     }
3114
3115     score_tab[start_i]= 0;
3116     survivor[0]= start_i;
3117     survivor_count= 1;
3118
3119     for(i=start_i; i<=last_non_zero; i++){
3120         int level_index, j, zero_distortion;
3121         int dct_coeff= FFABS(block[ scantable[i] ]);
3122         int best_score=256*256*256*120;
3123
3124         if (   s->dsp.fdct == fdct_ifast
3125 #ifndef FAAN_POSTSCALE
3126             || s->dsp.fdct == ff_faandct
3127 #endif
3128            )
3129             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3130         zero_distortion= dct_coeff*dct_coeff;
3131
3132         for(level_index=0; level_index < coeff_count[i]; level_index++){
3133             int distortion;
3134             int level= coeff[level_index][i];
3135             const int alevel= FFABS(level);
3136             int unquant_coeff;
3137
3138             assert(level);
3139
3140             if(s->out_format == FMT_H263){
3141                 unquant_coeff= alevel*qmul + qadd;
3142             }else{ //MPEG1
3143                 j= s->dsp.idct_permutation[ scantable[i] ]; //FIXME optimize
3144                 if(s->mb_intra){
3145                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3146                         unquant_coeff =   (unquant_coeff - 1) | 1;
3147                 }else{
3148                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3149                         unquant_coeff =   (unquant_coeff - 1) | 1;
3150                 }
3151                 unquant_coeff<<= 3;
3152             }
3153
3154             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3155             level+=64;
3156             if((level&(~127)) == 0){
3157                 for(j=survivor_count-1; j>=0; j--){
3158                     int run= i - survivor[j];
3159                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3160                     score += score_tab[i-run];
3161
3162                     if(score < best_score){
3163                         best_score= score;
3164                         run_tab[i+1]= run;
3165                         level_tab[i+1]= level-64;
3166                     }
3167                 }
3168
3169                 if(s->out_format == FMT_H263){
3170                     for(j=survivor_count-1; j>=0; j--){
3171                         int run= i - survivor[j];
3172                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3173                         score += score_tab[i-run];
3174                         if(score < last_score){
3175                             last_score= score;
3176                             last_run= run;
3177                             last_level= level-64;
3178                             last_i= i+1;
3179                         }
3180                     }
3181                 }
3182             }else{
3183                 distortion += esc_length*lambda;
3184                 for(j=survivor_count-1; j>=0; j--){
3185                     int run= i - survivor[j];
3186                     int score= distortion + score_tab[i-run];
3187
3188                     if(score < best_score){
3189                         best_score= score;
3190                         run_tab[i+1]= run;
3191                         level_tab[i+1]= level-64;
3192                     }
3193                 }
3194
3195                 if(s->out_format == FMT_H263){
3196                   for(j=survivor_count-1; j>=0; j--){
3197                         int run= i - survivor[j];
3198                         int score= distortion + score_tab[i-run];
3199                         if(score < last_score){
3200                             last_score= score;
3201                             last_run= run;
3202                             last_level= level-64;
3203                             last_i= i+1;
3204                         }
3205                     }
3206                 }
3207             }
3208         }
3209
3210         score_tab[i+1]= best_score;
3211
3212         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
3213         if(last_non_zero <= 27){
3214             for(; survivor_count; survivor_count--){
3215                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3216                     break;
3217             }
3218         }else{
3219             for(; survivor_count; survivor_count--){
3220                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3221                     break;
3222             }
3223         }
3224
3225         survivor[ survivor_count++ ]= i+1;
3226     }
3227
3228     if(s->out_format != FMT_H263){
3229         last_score= 256*256*256*120;
3230         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3231             int score= score_tab[i];
3232             if(i) score += lambda*2; //FIXME exacter?
3233
3234             if(score < last_score){
3235                 last_score= score;
3236                 last_i= i;
3237                 last_level= level_tab[i];
3238                 last_run= run_tab[i];
3239             }
3240         }
3241     }
3242
3243     s->coded_score[n] = last_score;
3244
3245     dc= FFABS(block[0]);
3246     last_non_zero= last_i - 1;
3247     memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3248
3249     if(last_non_zero < start_i)
3250         return last_non_zero;
3251
3252     if(last_non_zero == 0 && start_i == 0){
3253         int best_level= 0;
3254         int best_score= dc * dc;
3255
3256         for(i=0; i<coeff_count[0]; i++){
3257             int level= coeff[i][0];
3258             int alevel= FFABS(level);
3259             int unquant_coeff, score, distortion;
3260
3261             if(s->out_format == FMT_H263){
3262                     unquant_coeff= (alevel*qmul + qadd)>>3;
3263             }else{ //MPEG1
3264                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3265                     unquant_coeff =   (unquant_coeff - 1) | 1;
3266             }
3267             unquant_coeff = (unquant_coeff + 4) >> 3;
3268             unquant_coeff<<= 3 + 3;
3269
3270             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3271             level+=64;
3272             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3273             else                    score= distortion + esc_length*lambda;
3274
3275             if(score < best_score){
3276                 best_score= score;
3277                 best_level= level - 64;
3278             }
3279         }
3280         block[0]= best_level;
3281         s->coded_score[n] = best_score - dc*dc;
3282         if(best_level == 0) return -1;
3283         else                return last_non_zero;
3284     }
3285
3286     i= last_i;
3287     assert(last_level);
3288
3289     block[ perm_scantable[last_non_zero] ]= last_level;
3290     i -= last_run + 1;
3291
3292     for(; i>start_i; i -= run_tab[i] + 1){
3293         block[ perm_scantable[i-1] ]= level_tab[i];
3294     }
3295
3296     return last_non_zero;
3297 }
3298
3299 //#define REFINE_STATS 1
3300 static int16_t basis[64][64];
3301
3302 static void build_basis(uint8_t *perm){
3303     int i, j, x, y;
3304     emms_c();
3305     for(i=0; i<8; i++){
3306         for(j=0; j<8; j++){
3307             for(y=0; y<8; y++){
3308                 for(x=0; x<8; x++){
3309                     double s= 0.25*(1<<BASIS_SHIFT);
3310                     int index= 8*i + j;
3311                     int perm_index= perm[index];
3312                     if(i==0) s*= sqrt(0.5);
3313                     if(j==0) s*= sqrt(0.5);
3314                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3315                 }
3316             }
3317         }
3318     }
3319 }
3320
3321 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3322                         DCTELEM *block, int16_t *weight, DCTELEM *orig,
3323                         int n, int qscale){
3324     int16_t rem[64];
3325     LOCAL_ALIGNED_16(DCTELEM, d1, [64]);
3326     const uint8_t *scantable= s->intra_scantable.scantable;
3327     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3328 //    unsigned int threshold1, threshold2;
3329 //    int bias=0;
3330     int run_tab[65];
3331     int prev_run=0;
3332     int prev_level=0;
3333     int qmul, qadd, start_i, last_non_zero, i, dc;
3334     uint8_t * length;
3335     uint8_t * last_length;
3336     int lambda;
3337     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3338 #ifdef REFINE_STATS
3339 static int count=0;
3340 static int after_last=0;
3341 static int to_zero=0;
3342 static int from_zero=0;
3343 static int raise=0;
3344 static int lower=0;
3345 static int messed_sign=0;
3346 #endif
3347
3348     if(basis[0][0] == 0)
3349         build_basis(s->dsp.idct_permutation);
3350
3351     qmul= qscale*2;
3352     qadd= (qscale-1)|1;
3353     if (s->mb_intra) {
3354         if (!s->h263_aic) {
3355             if (n < 4)
3356                 q = s->y_dc_scale;
3357             else
3358                 q = s->c_dc_scale;
3359         } else{
3360             /* For AIC we skip quant/dequant of INTRADC */
3361             q = 1;
3362             qadd=0;
3363         }
3364         q <<= RECON_SHIFT-3;
3365         /* note: block[0] is assumed to be positive */
3366         dc= block[0]*q;
3367 //        block[0] = (block[0] + (q >> 1)) / q;