Do not fail when decoding invalid v410 files with odd width.
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of FFmpeg.
9  *
10  * FFmpeg is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * FFmpeg is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with FFmpeg; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include "libavutil/intmath.h"
31 #include "libavutil/mathematics.h"
32 #include "libavutil/opt.h"
33 #include "avcodec.h"
34 #include "dsputil.h"
35 #include "mpegvideo.h"
36 #include "mpegvideo_common.h"
37 #include "h263.h"
38 #include "mjpegenc.h"
39 #include "msmpeg4.h"
40 #include "faandct.h"
41 #include "thread.h"
42 #include "aandcttab.h"
43 #include "flv.h"
44 #include "mpeg4video.h"
45 #include "internal.h"
46 #include <limits.h>
47 #include "sp5x.h"
48
49 //#undef NDEBUG
50 //#include <assert.h>
51
52 static int encode_picture(MpegEncContext *s, int picture_number);
53 static int dct_quantize_refine(MpegEncContext *s, DCTELEM *block, int16_t *weight, DCTELEM *orig, int n, int qscale);
54 static int sse_mb(MpegEncContext *s);
55 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block);
56 static int dct_quantize_trellis_c(MpegEncContext *s, DCTELEM *block, int n, int qscale, int *overflow);
57
58 /* enable all paranoid tests for rounding, overflows, etc... */
59 //#define PARANOID
60
61 //#define DEBUG
62
63 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
64 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
65
66 void ff_convert_matrix(DSPContext *dsp, int (*qmat)[64],
67                        uint16_t (*qmat16)[2][64],
68                        const uint16_t *quant_matrix,
69                        int bias, int qmin, int qmax, int intra)
70 {
71     int qscale;
72     int shift = 0;
73
74     for (qscale = qmin; qscale <= qmax; qscale++) {
75         int i;
76         if (dsp->fdct == ff_jpeg_fdct_islow_8 ||
77             dsp->fdct == ff_jpeg_fdct_islow_10
78 #ifdef FAAN_POSTSCALE
79             || dsp->fdct == ff_faandct
80 #endif
81             ) {
82             for (i = 0; i < 64; i++) {
83                 const int j = dsp->idct_permutation[i];
84                 /* 16 <= qscale * quant_matrix[i] <= 7905
85                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
86                  *             19952 <=              x  <= 249205026
87                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
88                  *           3444240 >= (1 << 36) / (x) >= 275 */
89
90                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
91                                         (qscale * quant_matrix[j]));
92             }
93         } else if (dsp->fdct == fdct_ifast
94 #ifndef FAAN_POSTSCALE
95                    || dsp->fdct == ff_faandct
96 #endif
97                    ) {
98             for (i = 0; i < 64; i++) {
99                 const int j = dsp->idct_permutation[i];
100                 /* 16 <= qscale * quant_matrix[i] <= 7905
101                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
102                  *             19952 <=              x  <= 249205026
103                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
104                  *           3444240 >= (1 << 36) / (x) >= 275 */
105
106                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) /
107                                         (ff_aanscales[i] * qscale * quant_matrix[j]));
108             }
109         } else {
110             for (i = 0; i < 64; i++) {
111                 const int j = dsp->idct_permutation[i];
112                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
113                  * Assume x = qscale * quant_matrix[i]
114                  * So             16 <=              x  <= 7905
115                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
116                  * so          32768 >= (1 << 19) / (x) >= 67 */
117                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
118                                         (qscale * quant_matrix[j]));
119                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
120                 //                    (qscale * quant_matrix[i]);
121                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) /
122                                        (qscale * quant_matrix[j]);
123
124                 if (qmat16[qscale][0][i] == 0 ||
125                     qmat16[qscale][0][i] == 128 * 256)
126                     qmat16[qscale][0][i] = 128 * 256 - 1;
127                 qmat16[qscale][1][i] =
128                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
129                                 qmat16[qscale][0][i]);
130             }
131         }
132
133         for (i = intra; i < 64; i++) {
134             int64_t max = 8191;
135             if (dsp->fdct == fdct_ifast
136 #ifndef FAAN_POSTSCALE
137                 || dsp->fdct == ff_faandct
138 #endif
139                ) {
140                 max = (8191LL * ff_aanscales[i]) >> 14;
141             }
142             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
143                 shift++;
144             }
145         }
146     }
147     if (shift) {
148         av_log(NULL, AV_LOG_INFO,
149                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
150                QMAT_SHIFT - shift);
151     }
152 }
153
154 static inline void update_qscale(MpegEncContext *s)
155 {
156     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
157                 (FF_LAMBDA_SHIFT + 7);
158     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
159
160     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
161                  FF_LAMBDA_SHIFT;
162 }
163
164 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
165 {
166     int i;
167
168     if (matrix) {
169         put_bits(pb, 1, 1);
170         for (i = 0; i < 64; i++) {
171             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
172         }
173     } else
174         put_bits(pb, 1, 0);
175 }
176
177 /**
178  * init s->current_picture.qscale_table from s->lambda_table
179  */
180 void ff_init_qscale_tab(MpegEncContext *s)
181 {
182     int8_t * const qscale_table = s->current_picture.f.qscale_table;
183     int i;
184
185     for (i = 0; i < s->mb_num; i++) {
186         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
187         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
188         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
189                                                   s->avctx->qmax);
190     }
191 }
192
193 static void copy_picture_attributes(MpegEncContext *s,
194                                     AVFrame *dst,
195                                     AVFrame *src)
196 {
197     int i;
198
199     dst->pict_type              = src->pict_type;
200     dst->quality                = src->quality;
201     dst->coded_picture_number   = src->coded_picture_number;
202     dst->display_picture_number = src->display_picture_number;
203     //dst->reference              = src->reference;
204     dst->pts                    = src->pts;
205     dst->interlaced_frame       = src->interlaced_frame;
206     dst->top_field_first        = src->top_field_first;
207
208     if (s->avctx->me_threshold) {
209         if (!src->motion_val[0])
210             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_val not set!\n");
211         if (!src->mb_type)
212             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.mb_type not set!\n");
213         if (!src->ref_index[0])
214             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.ref_index not set!\n");
215         if (src->motion_subsample_log2 != dst->motion_subsample_log2)
216             av_log(s->avctx, AV_LOG_ERROR,
217                    "AVFrame.motion_subsample_log2 doesn't match! (%d!=%d)\n",
218                    src->motion_subsample_log2, dst->motion_subsample_log2);
219
220         memcpy(dst->mb_type, src->mb_type,
221                s->mb_stride * s->mb_height * sizeof(dst->mb_type[0]));
222
223         for (i = 0; i < 2; i++) {
224             int stride = ((16 * s->mb_width ) >>
225                           src->motion_subsample_log2) + 1;
226             int height = ((16 * s->mb_height) >> src->motion_subsample_log2);
227
228             if (src->motion_val[i] &&
229                 src->motion_val[i] != dst->motion_val[i]) {
230                 memcpy(dst->motion_val[i], src->motion_val[i],
231                        2 * stride * height * sizeof(int16_t));
232             }
233             if (src->ref_index[i] && src->ref_index[i] != dst->ref_index[i]) {
234                 memcpy(dst->ref_index[i], src->ref_index[i],
235                        s->mb_stride * 4 * s->mb_height * sizeof(int8_t));
236             }
237         }
238     }
239 }
240
241 static void update_duplicate_context_after_me(MpegEncContext *dst,
242                                               MpegEncContext *src)
243 {
244 #define COPY(a) dst->a= src->a
245     COPY(pict_type);
246     COPY(current_picture);
247     COPY(f_code);
248     COPY(b_code);
249     COPY(qscale);
250     COPY(lambda);
251     COPY(lambda2);
252     COPY(picture_in_gop_number);
253     COPY(gop_picture_number);
254     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
255     COPY(progressive_frame);    // FIXME don't set in encode_header
256     COPY(partitioned_frame);    // FIXME don't set in encode_header
257 #undef COPY
258 }
259
260 /**
261  * Set the given MpegEncContext to defaults for encoding.
262  * the changed fields will not depend upon the prior state of the MpegEncContext.
263  */
264 static void MPV_encode_defaults(MpegEncContext *s)
265 {
266     int i;
267     MPV_common_defaults(s);
268
269     for (i = -16; i < 16; i++) {
270         default_fcode_tab[i + MAX_MV] = 1;
271     }
272     s->me.mv_penalty = default_mv_penalty;
273     s->fcode_tab     = default_fcode_tab;
274 }
275
276 /* init video encoder */
277 av_cold int MPV_encode_init(AVCodecContext *avctx)
278 {
279     MpegEncContext *s = avctx->priv_data;
280     int i;
281     int chroma_h_shift, chroma_v_shift;
282
283     MPV_encode_defaults(s);
284
285     switch (avctx->codec_id) {
286     case CODEC_ID_MPEG2VIDEO:
287         if (avctx->pix_fmt != PIX_FMT_YUV420P &&
288             avctx->pix_fmt != PIX_FMT_YUV422P) {
289             av_log(avctx, AV_LOG_ERROR,
290                    "only YUV420 and YUV422 are supported\n");
291             return -1;
292         }
293         break;
294     case CODEC_ID_LJPEG:
295         if (avctx->pix_fmt != PIX_FMT_YUVJ420P &&
296             avctx->pix_fmt != PIX_FMT_YUVJ422P &&
297             avctx->pix_fmt != PIX_FMT_YUVJ444P &&
298             avctx->pix_fmt != PIX_FMT_BGRA     &&
299             ((avctx->pix_fmt != PIX_FMT_YUV420P &&
300               avctx->pix_fmt != PIX_FMT_YUV422P &&
301               avctx->pix_fmt != PIX_FMT_YUV444P) ||
302              avctx->strict_std_compliance > FF_COMPLIANCE_UNOFFICIAL)) {
303             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in LJPEG\n");
304             return -1;
305         }
306         break;
307     case CODEC_ID_MJPEG:
308     case CODEC_ID_AMV:
309         if (avctx->pix_fmt != PIX_FMT_YUVJ420P &&
310             avctx->pix_fmt != PIX_FMT_YUVJ422P &&
311             ((avctx->pix_fmt != PIX_FMT_YUV420P &&
312               avctx->pix_fmt != PIX_FMT_YUV422P) ||
313              avctx->strict_std_compliance > FF_COMPLIANCE_UNOFFICIAL)) {
314             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
315             return -1;
316         }
317         break;
318     default:
319         if (avctx->pix_fmt != PIX_FMT_YUV420P) {
320             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
321             return -1;
322         }
323     }
324
325     switch (avctx->pix_fmt) {
326     case PIX_FMT_YUVJ422P:
327     case PIX_FMT_YUV422P:
328         s->chroma_format = CHROMA_422;
329         break;
330     case PIX_FMT_YUVJ420P:
331     case PIX_FMT_YUV420P:
332     default:
333         s->chroma_format = CHROMA_420;
334         break;
335     }
336
337     s->bit_rate = avctx->bit_rate;
338     s->width    = avctx->width;
339     s->height   = avctx->height;
340     if (avctx->gop_size > 600 &&
341         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
342         av_log(avctx, AV_LOG_ERROR,
343                "Warning keyframe interval too large! reducing it ...\n");
344         avctx->gop_size = 600;
345     }
346     s->gop_size     = avctx->gop_size;
347     s->avctx        = avctx;
348     s->flags        = avctx->flags;
349     s->flags2       = avctx->flags2;
350     s->max_b_frames = avctx->max_b_frames;
351     s->codec_id     = avctx->codec->id;
352     s->luma_elim_threshold   = avctx->luma_elim_threshold;
353     s->chroma_elim_threshold = avctx->chroma_elim_threshold;
354     s->strict_std_compliance = avctx->strict_std_compliance;
355 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
356     if (avctx->flags & CODEC_FLAG_PART)
357         s->data_partitioning = 1;
358 #endif
359     s->quarter_sample     = (avctx->flags & CODEC_FLAG_QPEL) != 0;
360     s->mpeg_quant         = avctx->mpeg_quant;
361     s->rtp_mode           = !!avctx->rtp_payload_size;
362     s->intra_dc_precision = avctx->intra_dc_precision;
363     s->user_specified_pts = AV_NOPTS_VALUE;
364
365     if (s->gop_size <= 1) {
366         s->intra_only = 1;
367         s->gop_size   = 12;
368     } else {
369         s->intra_only = 0;
370     }
371
372     s->me_method = avctx->me_method;
373
374     /* Fixed QSCALE */
375     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
376
377     s->adaptive_quant = (s->avctx->lumi_masking ||
378                          s->avctx->dark_masking ||
379                          s->avctx->temporal_cplx_masking ||
380                          s->avctx->spatial_cplx_masking  ||
381                          s->avctx->p_masking      ||
382                          s->avctx->border_masking ||
383                          (s->flags & CODEC_FLAG_QP_RD)) &&
384                         !s->fixed_qscale;
385
386     s->loop_filter      = !!(s->flags & CODEC_FLAG_LOOP_FILTER);
387 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
388     s->alternate_scan   = !!(s->flags  & CODEC_FLAG_ALT_SCAN);
389     s->intra_vlc_format = !!(s->flags2 & CODEC_FLAG2_INTRA_VLC);
390     s->q_scale_type     = !!(s->flags2 & CODEC_FLAG2_NON_LINEAR_QUANT);
391     s->obmc             = !!(s->flags  & CODEC_FLAG_OBMC);
392 #endif
393
394     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
395         av_log(avctx, AV_LOG_ERROR,
396                "a vbv buffer size is needed, "
397                "for encoding with a maximum bitrate\n");
398         return -1;
399     }
400
401     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
402         av_log(avctx, AV_LOG_INFO,
403                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
404     }
405
406     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
407         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
408         return -1;
409     }
410
411     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
412         av_log(avctx, AV_LOG_ERROR, "bitrate above max bitrate\n");
413         return -1;
414     }
415
416     if (avctx->rc_max_rate &&
417         avctx->rc_max_rate == avctx->bit_rate &&
418         avctx->rc_max_rate != avctx->rc_min_rate) {
419         av_log(avctx, AV_LOG_INFO,
420                "impossible bitrate constraints, this will fail\n");
421     }
422
423     if (avctx->rc_buffer_size &&
424         avctx->bit_rate * (int64_t)avctx->time_base.num >
425             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
426         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
427         return -1;
428     }
429
430     if (!s->fixed_qscale &&
431         avctx->bit_rate * av_q2d(avctx->time_base) >
432             avctx->bit_rate_tolerance) {
433         av_log(avctx, AV_LOG_ERROR,
434                "bitrate tolerance too small for bitrate\n");
435         return -1;
436     }
437
438     if (s->avctx->rc_max_rate &&
439         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
440         (s->codec_id == CODEC_ID_MPEG1VIDEO ||
441          s->codec_id == CODEC_ID_MPEG2VIDEO) &&
442         90000LL * (avctx->rc_buffer_size - 1) >
443             s->avctx->rc_max_rate * 0xFFFFLL) {
444         av_log(avctx, AV_LOG_INFO,
445                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
446                "specified vbv buffer is too large for the given bitrate!\n");
447     }
448
449     if ((s->flags & CODEC_FLAG_4MV)  && s->codec_id != CODEC_ID_MPEG4 &&
450         s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P &&
451         s->codec_id != CODEC_ID_FLV1) {
452         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
453         return -1;
454     }
455
456     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
457         av_log(avctx, AV_LOG_ERROR,
458                "OBMC is only supported with simple mb decision\n");
459         return -1;
460     }
461
462 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
463     if (s->obmc && s->codec_id != CODEC_ID_H263 &&
464         s->codec_id != CODEC_ID_H263P) {
465         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with H263(+)\n");
466         return -1;
467     }
468 #endif
469
470     if (s->quarter_sample && s->codec_id != CODEC_ID_MPEG4) {
471         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
472         return -1;
473     }
474
475 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
476     if (s->data_partitioning && s->codec_id != CODEC_ID_MPEG4) {
477         av_log(avctx, AV_LOG_ERROR,
478                "data partitioning not supported by codec\n");
479         return -1;
480     }
481 #endif
482
483     if (s->max_b_frames                    &&
484         s->codec_id != CODEC_ID_MPEG4      &&
485         s->codec_id != CODEC_ID_MPEG1VIDEO &&
486         s->codec_id != CODEC_ID_MPEG2VIDEO) {
487         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
488         return -1;
489     }
490
491     if ((s->codec_id == CODEC_ID_MPEG4 ||
492          s->codec_id == CODEC_ID_H263  ||
493          s->codec_id == CODEC_ID_H263P) &&
494         (avctx->sample_aspect_ratio.num > 255 ||
495          avctx->sample_aspect_ratio.den > 255)) {
496         av_log(avctx, AV_LOG_WARNING,
497                "Invalid pixel aspect ratio %i/%i, limit is 255/255 reducing\n",
498                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
499         av_reduce(&avctx->sample_aspect_ratio.num, &avctx->sample_aspect_ratio.den,
500                    avctx->sample_aspect_ratio.num,  avctx->sample_aspect_ratio.den, 255);
501     }
502
503     if ((s->flags & (CODEC_FLAG_INTERLACED_DCT | CODEC_FLAG_INTERLACED_ME |
504                      CODEC_FLAG_ALT_SCAN)) &&
505         s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG2VIDEO) {
506         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
507         return -1;
508     }
509
510     // FIXME mpeg2 uses that too
511     if (s->mpeg_quant && s->codec_id != CODEC_ID_MPEG4) {
512         av_log(avctx, AV_LOG_ERROR,
513                "mpeg2 style quantization not supported by codec\n");
514         return -1;
515     }
516
517     if ((s->flags & CODEC_FLAG_CBP_RD) && !avctx->trellis) {
518         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
519         return -1;
520     }
521
522     if ((s->flags & CODEC_FLAG_QP_RD) &&
523         s->avctx->mb_decision != FF_MB_DECISION_RD) {
524         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
525         return -1;
526     }
527
528     if (s->avctx->scenechange_threshold < 1000000000 &&
529         (s->flags & CODEC_FLAG_CLOSED_GOP)) {
530         av_log(avctx, AV_LOG_ERROR,
531                "closed gop with scene change detection are not supported yet, "
532                "set threshold to 1000000000\n");
533         return -1;
534     }
535
536     if ((s->flags2 & CODEC_FLAG2_INTRA_VLC) &&
537         s->codec_id != CODEC_ID_MPEG2VIDEO) {
538         av_log(avctx, AV_LOG_ERROR,
539                "intra vlc table not supported by codec\n");
540         return -1;
541     }
542
543     if (s->flags & CODEC_FLAG_LOW_DELAY) {
544         if (s->codec_id != CODEC_ID_MPEG2VIDEO) {
545             av_log(avctx, AV_LOG_ERROR,
546                   "low delay forcing is only available for mpeg2\n");
547             return -1;
548         }
549         if (s->max_b_frames != 0) {
550             av_log(avctx, AV_LOG_ERROR,
551                    "b frames cannot be used with low delay\n");
552             return -1;
553         }
554     }
555
556     if (s->q_scale_type == 1) {
557 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
558         if (s->codec_id != CODEC_ID_MPEG2VIDEO) {
559             av_log(avctx, AV_LOG_ERROR,
560                    "non linear quant is only available for mpeg2\n");
561             return -1;
562         }
563 #endif
564         if (avctx->qmax > 12) {
565             av_log(avctx, AV_LOG_ERROR,
566                    "non linear quant only supports qmax <= 12 currently\n");
567             return -1;
568         }
569     }
570
571     if (s->avctx->thread_count > 1         &&
572         s->codec_id != CODEC_ID_MPEG4      &&
573         s->codec_id != CODEC_ID_MPEG1VIDEO &&
574         s->codec_id != CODEC_ID_MPEG2VIDEO &&
575         (s->codec_id != CODEC_ID_H263P ||
576          !(s->flags & CODEC_FLAG_H263P_SLICE_STRUCT))) {
577         av_log(avctx, AV_LOG_ERROR,
578                "multi threaded encoding not supported by codec\n");
579         return -1;
580     }
581
582     if (s->avctx->thread_count < 1) {
583         av_log(avctx, AV_LOG_ERROR,
584                "automatic thread number detection not supported by codec, "
585                "patch welcome\n");
586         return -1;
587     }
588
589     if (s->avctx->thread_count > 1)
590         s->rtp_mode = 1;
591
592     if (!avctx->time_base.den || !avctx->time_base.num) {
593         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
594         return -1;
595     }
596
597     i = (INT_MAX / 2 + 128) >> 8;
598     if (avctx->me_threshold >= i) {
599         av_log(avctx, AV_LOG_ERROR, "me_threshold too large, max is %d\n",
600                i - 1);
601         return -1;
602     }
603     if (avctx->mb_threshold >= i) {
604         av_log(avctx, AV_LOG_ERROR, "mb_threshold too large, max is %d\n",
605                i - 1);
606         return -1;
607     }
608
609     if (avctx->b_frame_strategy && (avctx->flags & CODEC_FLAG_PASS2)) {
610         av_log(avctx, AV_LOG_INFO,
611                "notice: b_frame_strategy only affects the first pass\n");
612         avctx->b_frame_strategy = 0;
613     }
614
615     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
616     if (i > 1) {
617         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
618         avctx->time_base.den /= i;
619         avctx->time_base.num /= i;
620         //return -1;
621     }
622
623     if (s->mpeg_quant || s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO || s->codec_id == CODEC_ID_MJPEG || s->codec_id==CODEC_ID_AMV) {
624         // (a + x * 3 / 8) / x
625         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
626         s->inter_quant_bias = 0;
627     } else {
628         s->intra_quant_bias = 0;
629         // (a - x / 4) / x
630         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
631     }
632
633     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
634         s->intra_quant_bias = avctx->intra_quant_bias;
635     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
636         s->inter_quant_bias = avctx->inter_quant_bias;
637
638     av_log(avctx, AV_LOG_DEBUG, "intra_quant_bias = %d inter_quant_bias = %d\n",s->intra_quant_bias,s->inter_quant_bias);
639
640     avcodec_get_chroma_sub_sample(avctx->pix_fmt, &chroma_h_shift,
641                                   &chroma_v_shift);
642
643     if (avctx->codec_id == CODEC_ID_MPEG4 &&
644         s->avctx->time_base.den > (1 << 16) - 1) {
645         av_log(avctx, AV_LOG_ERROR,
646                "timebase %d/%d not supported by MPEG 4 standard, "
647                "the maximum admitted value for the timebase denominator "
648                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
649                (1 << 16) - 1);
650         return -1;
651     }
652     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
653
654     switch (avctx->codec->id) {
655     case CODEC_ID_MPEG1VIDEO:
656         s->out_format = FMT_MPEG1;
657         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
658         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
659         break;
660     case CODEC_ID_MPEG2VIDEO:
661         s->out_format = FMT_MPEG1;
662         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
663         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
664         s->rtp_mode   = 1;
665         break;
666     case CODEC_ID_LJPEG:
667     case CODEC_ID_MJPEG:
668     case CODEC_ID_AMV:
669         s->out_format = FMT_MJPEG;
670         s->intra_only = 1; /* force intra only for jpeg */
671         if (avctx->codec->id == CODEC_ID_LJPEG && avctx->pix_fmt   == PIX_FMT_BGRA) {
672             s->mjpeg_vsample[0] = s->mjpeg_hsample[0] =
673             s->mjpeg_vsample[1] = s->mjpeg_hsample[1] =
674             s->mjpeg_vsample[2] = s->mjpeg_hsample[2] = 1;
675         } else {
676             s->mjpeg_vsample[0] = 2;
677             s->mjpeg_vsample[1] = 2 >> chroma_v_shift;
678             s->mjpeg_vsample[2] = 2 >> chroma_v_shift;
679             s->mjpeg_hsample[0] = 2;
680             s->mjpeg_hsample[1] = 2 >> chroma_h_shift;
681             s->mjpeg_hsample[2] = 2 >> chroma_h_shift;
682         }
683         if (!(CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER) ||
684             ff_mjpeg_encode_init(s) < 0)
685             return -1;
686         avctx->delay = 0;
687         s->low_delay = 1;
688         break;
689     case CODEC_ID_H261:
690         if (!CONFIG_H261_ENCODER)
691             return -1;
692         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
693             av_log(avctx, AV_LOG_ERROR,
694                    "The specified picture size of %dx%d is not valid for the "
695                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
696                     s->width, s->height);
697             return -1;
698         }
699         s->out_format = FMT_H261;
700         avctx->delay  = 0;
701         s->low_delay  = 1;
702         break;
703     case CODEC_ID_H263:
704         if (!CONFIG_H263_ENCODER)
705             return -1;
706         if (ff_match_2uint16(h263_format, FF_ARRAY_ELEMS(h263_format),
707                              s->width, s->height) == 8) {
708             av_log(avctx, AV_LOG_ERROR,
709                    "The specified picture size of %dx%d is not valid for "
710                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
711                    "352x288, 704x576, and 1408x1152. "
712                    "Try H.263+.\n", s->width, s->height);
713             return -1;
714         }
715         s->out_format = FMT_H263;
716         avctx->delay  = 0;
717         s->low_delay  = 1;
718         break;
719     case CODEC_ID_H263P:
720         s->out_format = FMT_H263;
721         s->h263_plus  = 1;
722         /* Fx */
723 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
724         if (avctx->flags & CODEC_FLAG_H263P_UMV)
725             s->umvplus = 1;
726         if (avctx->flags & CODEC_FLAG_H263P_AIV)
727             s->alt_inter_vlc = 1;
728         if (avctx->flags & CODEC_FLAG_H263P_SLICE_STRUCT)
729             s->h263_slice_structured = 1;
730 #endif
731         s->h263_aic        = (avctx->flags & CODEC_FLAG_AC_PRED) ? 1 : 0;
732         s->modified_quant  = s->h263_aic;
733         s->loop_filter     = (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
734         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
735
736         /* /Fx */
737         /* These are just to be sure */
738         avctx->delay = 0;
739         s->low_delay = 1;
740         break;
741     case CODEC_ID_FLV1:
742         s->out_format      = FMT_H263;
743         s->h263_flv        = 2; /* format = 1; 11-bit codes */
744         s->unrestricted_mv = 1;
745         s->rtp_mode  = 0; /* don't allow GOB */
746         avctx->delay = 0;
747         s->low_delay = 1;
748         break;
749     case CODEC_ID_RV10:
750         s->out_format = FMT_H263;
751         avctx->delay  = 0;
752         s->low_delay  = 1;
753         break;
754     case CODEC_ID_RV20:
755         s->out_format      = FMT_H263;
756         avctx->delay       = 0;
757         s->low_delay       = 1;
758         s->modified_quant  = 1;
759         s->h263_aic        = 1;
760         s->h263_plus       = 1;
761         s->loop_filter     = 1;
762         s->unrestricted_mv = 0;
763         break;
764     case CODEC_ID_MPEG4:
765         s->out_format      = FMT_H263;
766         s->h263_pred       = 1;
767         s->unrestricted_mv = 1;
768         s->low_delay       = s->max_b_frames ? 0 : 1;
769         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
770         break;
771     case CODEC_ID_MSMPEG4V2:
772         s->out_format      = FMT_H263;
773         s->h263_pred       = 1;
774         s->unrestricted_mv = 1;
775         s->msmpeg4_version = 2;
776         avctx->delay       = 0;
777         s->low_delay       = 1;
778         break;
779     case CODEC_ID_MSMPEG4V3:
780         s->out_format        = FMT_H263;
781         s->h263_pred         = 1;
782         s->unrestricted_mv   = 1;
783         s->msmpeg4_version   = 3;
784         s->flipflop_rounding = 1;
785         avctx->delay         = 0;
786         s->low_delay         = 1;
787         break;
788     case CODEC_ID_WMV1:
789         s->out_format        = FMT_H263;
790         s->h263_pred         = 1;
791         s->unrestricted_mv   = 1;
792         s->msmpeg4_version   = 4;
793         s->flipflop_rounding = 1;
794         avctx->delay         = 0;
795         s->low_delay         = 1;
796         break;
797     case CODEC_ID_WMV2:
798         s->out_format        = FMT_H263;
799         s->h263_pred         = 1;
800         s->unrestricted_mv   = 1;
801         s->msmpeg4_version   = 5;
802         s->flipflop_rounding = 1;
803         avctx->delay         = 0;
804         s->low_delay         = 1;
805         break;
806     default:
807         return -1;
808     }
809
810     avctx->has_b_frames = !s->low_delay;
811
812     s->encoding = 1;
813
814     s->progressive_frame    =
815     s->progressive_sequence = !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT |
816                                                 CODEC_FLAG_INTERLACED_ME  |
817                                                 CODEC_FLAG_ALT_SCAN));
818
819     /* init */
820     if (MPV_common_init(s) < 0)
821         return -1;
822
823     if (!s->dct_quantize)
824         s->dct_quantize = dct_quantize_c;
825     if (!s->denoise_dct)
826         s->denoise_dct  = denoise_dct_c;
827     s->fast_dct_quantize = s->dct_quantize;
828     if (avctx->trellis)
829         s->dct_quantize  = dct_quantize_trellis_c;
830
831     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
832         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
833
834     s->quant_precision = 5;
835
836     ff_set_cmp(&s->dsp, s->dsp.ildct_cmp, s->avctx->ildct_cmp);
837     ff_set_cmp(&s->dsp, s->dsp.frame_skip_cmp, s->avctx->frame_skip_cmp);
838
839     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
840         ff_h261_encode_init(s);
841     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
842         h263_encode_init(s);
843     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
844         ff_msmpeg4_encode_init(s);
845     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
846         && s->out_format == FMT_MPEG1)
847         ff_mpeg1_encode_init(s);
848
849     /* init q matrix */
850     for (i = 0; i < 64; i++) {
851         int j = s->dsp.idct_permutation[i];
852         if (CONFIG_MPEG4_ENCODER && s->codec_id == CODEC_ID_MPEG4 &&
853             s->mpeg_quant) {
854             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
855             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
856         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
857             s->intra_matrix[j] =
858             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
859         } else {
860             /* mpeg1/2 */
861             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
862             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
863         }
864         if (s->avctx->intra_matrix)
865             s->intra_matrix[j] = s->avctx->intra_matrix[i];
866         if (s->avctx->inter_matrix)
867             s->inter_matrix[j] = s->avctx->inter_matrix[i];
868     }
869
870     /* precompute matrix */
871     /* for mjpeg, we do include qscale in the matrix */
872     if (s->out_format != FMT_MJPEG) {
873         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
874                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
875                           31, 1);
876         ff_convert_matrix(&s->dsp, s->q_inter_matrix, s->q_inter_matrix16,
877                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
878                           31, 0);
879     }
880
881     if (ff_rate_control_init(s) < 0)
882         return -1;
883
884     return 0;
885 }
886
887 av_cold int MPV_encode_end(AVCodecContext *avctx)
888 {
889     MpegEncContext *s = avctx->priv_data;
890
891     ff_rate_control_uninit(s);
892
893     MPV_common_end(s);
894     if ((CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER) &&
895         s->out_format == FMT_MJPEG)
896         ff_mjpeg_encode_close(s);
897
898     av_freep(&avctx->extradata);
899
900     return 0;
901 }
902
903 static int get_sae(uint8_t *src, int ref, int stride)
904 {
905     int x,y;
906     int acc = 0;
907
908     for (y = 0; y < 16; y++) {
909         for (x = 0; x < 16; x++) {
910             acc += FFABS(src[x + y * stride] - ref);
911         }
912     }
913
914     return acc;
915 }
916
917 static int get_intra_count(MpegEncContext *s, uint8_t *src,
918                            uint8_t *ref, int stride)
919 {
920     int x, y, w, h;
921     int acc = 0;
922
923     w = s->width  & ~15;
924     h = s->height & ~15;
925
926     for (y = 0; y < h; y += 16) {
927         for (x = 0; x < w; x += 16) {
928             int offset = x + y * stride;
929             int sad  = s->dsp.sad[0](NULL, src + offset, ref + offset, stride,
930                                      16);
931             int mean = (s->dsp.pix_sum(src + offset, stride) + 128) >> 8;
932             int sae  = get_sae(src + offset, mean, stride);
933
934             acc += sae + 500 < sad;
935         }
936     }
937     return acc;
938 }
939
940
941 static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg)
942 {
943     AVFrame *pic = NULL;
944     int64_t pts;
945     int i;
946     const int encoding_delay = s->max_b_frames;
947     int direct = 1;
948
949     if (pic_arg) {
950         pts = pic_arg->pts;
951         pic_arg->display_picture_number = s->input_picture_number++;
952
953         if (pts != AV_NOPTS_VALUE) {
954             if (s->user_specified_pts != AV_NOPTS_VALUE) {
955                 int64_t time = pts;
956                 int64_t last = s->user_specified_pts;
957
958                 if (time <= last) {
959                     av_log(s->avctx, AV_LOG_ERROR,
960                            "Error, Invalid timestamp=%"PRId64", "
961                            "last=%"PRId64"\n", pts, s->user_specified_pts);
962                     return -1;
963                 }
964             }
965             s->user_specified_pts = pts;
966         } else {
967             if (s->user_specified_pts != AV_NOPTS_VALUE) {
968                 s->user_specified_pts =
969                 pts = s->user_specified_pts + 1;
970                 av_log(s->avctx, AV_LOG_INFO,
971                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
972                        pts);
973             } else {
974                 pts = pic_arg->display_picture_number;
975             }
976         }
977     }
978
979   if (pic_arg) {
980     if (encoding_delay && !(s->flags & CODEC_FLAG_INPUT_PRESERVED))
981         direct = 0;
982     if (pic_arg->linesize[0] != s->linesize)
983         direct = 0;
984     if (pic_arg->linesize[1] != s->uvlinesize)
985         direct = 0;
986     if (pic_arg->linesize[2] != s->uvlinesize)
987         direct = 0;
988
989     //av_log(AV_LOG_DEBUG, "%d %d %d %d\n",pic_arg->linesize[0],
990     //       pic_arg->linesize[1], s->linesize, s->uvlinesize);
991
992     if (direct) {
993         i = ff_find_unused_picture(s, 1);
994         if (i < 0)
995             return i;
996
997         pic = (AVFrame *) &s->picture[i];
998         pic->reference = 3;
999
1000         for (i = 0; i < 4; i++) {
1001             pic->data[i]     = pic_arg->data[i];
1002             pic->linesize[i] = pic_arg->linesize[i];
1003         }
1004         if (ff_alloc_picture(s, (Picture *) pic, 1) < 0) {
1005             return -1;
1006         }
1007     } else {
1008         i = ff_find_unused_picture(s, 0);
1009         if (i < 0)
1010             return i;
1011
1012         pic = (AVFrame *) &s->picture[i];
1013         pic->reference = 3;
1014
1015         if (ff_alloc_picture(s, (Picture *) pic, 0) < 0) {
1016             return -1;
1017         }
1018
1019         if (pic->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1020             pic->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1021             pic->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1022             // empty
1023         } else {
1024             int h_chroma_shift, v_chroma_shift;
1025             avcodec_get_chroma_sub_sample(s->avctx->pix_fmt, &h_chroma_shift,
1026                                           &v_chroma_shift);
1027
1028             for (i = 0; i < 3; i++) {
1029                 int src_stride = pic_arg->linesize[i];
1030                 int dst_stride = i ? s->uvlinesize : s->linesize;
1031                 int h_shift = i ? h_chroma_shift : 0;
1032                 int v_shift = i ? v_chroma_shift : 0;
1033                 int w = s->width  >> h_shift;
1034                 int h = s->height >> v_shift;
1035                 uint8_t *src = pic_arg->data[i];
1036                 uint8_t *dst = pic->data[i];
1037
1038                 if(s->codec_id == CODEC_ID_AMV && !(s->avctx->flags & CODEC_FLAG_EMU_EDGE)){
1039                     h= ((s->height+15)/16*16)>>v_shift;
1040                 }
1041
1042                 if (!s->avctx->rc_buffer_size)
1043                     dst += INPLACE_OFFSET;
1044
1045                 if (src_stride == dst_stride)
1046                     memcpy(dst, src, src_stride * h);
1047                 else {
1048                     while (h--) {
1049                         memcpy(dst, src, w);
1050                         dst += dst_stride;
1051                         src += src_stride;
1052                     }
1053                 }
1054             }
1055         }
1056     }
1057     copy_picture_attributes(s, pic, pic_arg);
1058     pic->pts = pts; // we set this here to avoid modifiying pic_arg
1059   }
1060
1061     /* shift buffer entries */
1062     for (i = 1; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1063         s->input_picture[i - 1] = s->input_picture[i];
1064
1065     s->input_picture[encoding_delay] = (Picture*) pic;
1066
1067     return 0;
1068 }
1069
1070 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1071 {
1072     int x, y, plane;
1073     int score = 0;
1074     int64_t score64 = 0;
1075
1076     for (plane = 0; plane < 3; plane++) {
1077         const int stride = p->f.linesize[plane];
1078         const int bw = plane ? 1 : 2;
1079         for (y = 0; y < s->mb_height * bw; y++) {
1080             for (x = 0; x < s->mb_width * bw; x++) {
1081                 int off = p->f.type == FF_BUFFER_TYPE_SHARED ? 0 : 16;
1082                 uint8_t *dptr = p->f.data[plane] + 8 * (x + y * stride) + off;
1083                 uint8_t *rptr = ref->f.data[plane] + 8 * (x + y * stride);
1084                 int v   = s->dsp.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1085
1086                 switch (s->avctx->frame_skip_exp) {
1087                 case 0: score    =  FFMAX(score, v);          break;
1088                 case 1: score   += FFABS(v);                  break;
1089                 case 2: score   += v * v;                     break;
1090                 case 3: score64 += FFABS(v * v * (int64_t)v); break;
1091                 case 4: score64 += v * v * (int64_t)(v * v);  break;
1092                 }
1093             }
1094         }
1095     }
1096
1097     if (score)
1098         score64 = score;
1099
1100     if (score64 < s->avctx->frame_skip_threshold)
1101         return 1;
1102     if (score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda) >> 8))
1103         return 1;
1104     return 0;
1105 }
1106
1107 static int estimate_best_b_count(MpegEncContext *s)
1108 {
1109     AVCodec *codec = avcodec_find_encoder(s->avctx->codec_id);
1110     AVCodecContext *c = avcodec_alloc_context3(NULL);
1111     AVFrame input[FF_MAX_B_FRAMES + 2];
1112     const int scale = s->avctx->brd_scale;
1113     int i, j, out_size, p_lambda, b_lambda, lambda2;
1114     int outbuf_size  = s->width * s->height; // FIXME
1115     uint8_t *outbuf  = av_malloc(outbuf_size);
1116     int64_t best_rd  = INT64_MAX;
1117     int best_b_count = -1;
1118
1119     assert(scale >= 0 && scale <= 3);
1120
1121     //emms_c();
1122     //s->next_picture_ptr->quality;
1123     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1124     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1125     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1126     if (!b_lambda) // FIXME we should do this somewhere else
1127         b_lambda = p_lambda;
1128     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1129                FF_LAMBDA_SHIFT;
1130
1131     c->width        = s->width  >> scale;
1132     c->height       = s->height >> scale;
1133     c->flags        = CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR |
1134                       CODEC_FLAG_INPUT_PRESERVED /*| CODEC_FLAG_EMU_EDGE*/;
1135     c->flags       |= s->avctx->flags & CODEC_FLAG_QPEL;
1136     c->mb_decision  = s->avctx->mb_decision;
1137     c->me_cmp       = s->avctx->me_cmp;
1138     c->mb_cmp       = s->avctx->mb_cmp;
1139     c->me_sub_cmp   = s->avctx->me_sub_cmp;
1140     c->pix_fmt      = PIX_FMT_YUV420P;
1141     c->time_base    = s->avctx->time_base;
1142     c->max_b_frames = s->max_b_frames;
1143
1144     if (avcodec_open2(c, codec, NULL) < 0)
1145         return -1;
1146
1147     for(i=0; i<s->max_b_frames+2; i++){
1148         int ysize= c->width*c->height;
1149         int csize= (c->width/2)*(c->height/2);
1150         Picture pre_input, *pre_input_ptr= i ? s->input_picture[i-1] : s->next_picture_ptr;
1151
1152         avcodec_get_frame_defaults(&input[i]);
1153         input[i].data[0]= av_malloc(ysize + 2*csize);
1154         input[i].data[1]= input[i].data[0] + ysize;
1155         input[i].data[2]= input[i].data[1] + csize;
1156         input[i].linesize[0]= c->width;
1157         input[i].linesize[1]=
1158         input[i].linesize[2]= c->width/2;
1159
1160         if(pre_input_ptr && (!i || s->input_picture[i-1])) {
1161             pre_input= *pre_input_ptr;
1162
1163             if (pre_input.f.type != FF_BUFFER_TYPE_SHARED && i) {
1164                 pre_input.f.data[0] += INPLACE_OFFSET;
1165                 pre_input.f.data[1] += INPLACE_OFFSET;
1166                 pre_input.f.data[2] += INPLACE_OFFSET;
1167             }
1168
1169             s->dsp.shrink[scale](input[i].data[0], input[i].linesize[0], pre_input.f.data[0], pre_input.f.linesize[0], c->width,      c->height);
1170             s->dsp.shrink[scale](input[i].data[1], input[i].linesize[1], pre_input.f.data[1], pre_input.f.linesize[1], c->width >> 1, c->height >> 1);
1171             s->dsp.shrink[scale](input[i].data[2], input[i].linesize[2], pre_input.f.data[2], pre_input.f.linesize[2], c->width >> 1, c->height >> 1);
1172         }
1173     }
1174
1175     for(j=0; j<s->max_b_frames+1; j++){
1176         int64_t rd=0;
1177
1178         if(!s->input_picture[j])
1179             break;
1180
1181         c->error[0]= c->error[1]= c->error[2]= 0;
1182
1183         input[0].pict_type= AV_PICTURE_TYPE_I;
1184         input[0].quality= 1 * FF_QP2LAMBDA;
1185         out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[0]);
1186 //        rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1187
1188         for(i=0; i<s->max_b_frames+1; i++){
1189             int is_p= i % (j+1) == j || i==s->max_b_frames;
1190
1191             input[i+1].pict_type= is_p ? AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1192             input[i+1].quality= is_p ? p_lambda : b_lambda;
1193             out_size = avcodec_encode_video(c, outbuf, outbuf_size, &input[i+1]);
1194             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1195         }
1196
1197         /* get the delayed frames */
1198         while(out_size){
1199             out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL);
1200             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1201         }
1202
1203         rd += c->error[0] + c->error[1] + c->error[2];
1204
1205         if(rd < best_rd){
1206             best_rd= rd;
1207             best_b_count= j;
1208         }
1209     }
1210
1211     av_freep(&outbuf);
1212     avcodec_close(c);
1213     av_freep(&c);
1214
1215     for(i=0; i<s->max_b_frames+2; i++){
1216         av_freep(&input[i].data[0]);
1217     }
1218
1219     return best_b_count;
1220 }
1221
1222 static int select_input_picture(MpegEncContext *s){
1223     int i;
1224
1225     for(i=1; i<MAX_PICTURE_COUNT; i++)
1226         s->reordered_input_picture[i-1]= s->reordered_input_picture[i];
1227     s->reordered_input_picture[MAX_PICTURE_COUNT-1]= NULL;
1228
1229     /* set next picture type & ordering */
1230     if(s->reordered_input_picture[0]==NULL && s->input_picture[0]){
1231         if(/*s->picture_in_gop_number >= s->gop_size ||*/ s->next_picture_ptr==NULL || s->intra_only){
1232             s->reordered_input_picture[0]= s->input_picture[0];
1233             s->reordered_input_picture[0]->f.pict_type = AV_PICTURE_TYPE_I;
1234             s->reordered_input_picture[0]->f.coded_picture_number = s->coded_picture_number++;
1235         }else{
1236             int b_frames;
1237
1238             if(s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor){
1239                 if(s->picture_in_gop_number < s->gop_size && skip_check(s, s->input_picture[0], s->next_picture_ptr)){
1240                 //FIXME check that te gop check above is +-1 correct
1241 //av_log(NULL, AV_LOG_DEBUG, "skip %p %"PRId64"\n", s->input_picture[0]->f.data[0], s->input_picture[0]->pts);
1242
1243                     if (s->input_picture[0]->f.type == FF_BUFFER_TYPE_SHARED) {
1244                         for(i=0; i<4; i++)
1245                             s->input_picture[0]->f.data[i] = NULL;
1246                         s->input_picture[0]->f.type = 0;
1247                     }else{
1248                         assert(   s->input_picture[0]->f.type == FF_BUFFER_TYPE_USER
1249                                || s->input_picture[0]->f.type == FF_BUFFER_TYPE_INTERNAL);
1250
1251                         s->avctx->release_buffer(s->avctx, (AVFrame*)s->input_picture[0]);
1252                     }
1253
1254                     emms_c();
1255                     ff_vbv_update(s, 0);
1256
1257                     goto no_output_pic;
1258                 }
1259             }
1260
1261             if(s->flags&CODEC_FLAG_PASS2){
1262                 for(i=0; i<s->max_b_frames+1; i++){
1263                     int pict_num = s->input_picture[0]->f.display_picture_number + i;
1264
1265                     if(pict_num >= s->rc_context.num_entries)
1266                         break;
1267                     if(!s->input_picture[i]){
1268                         s->rc_context.entry[pict_num-1].new_pict_type = AV_PICTURE_TYPE_P;
1269                         break;
1270                     }
1271
1272                     s->input_picture[i]->f.pict_type =
1273                         s->rc_context.entry[pict_num].new_pict_type;
1274                 }
1275             }
1276
1277             if(s->avctx->b_frame_strategy==0){
1278                 b_frames= s->max_b_frames;
1279                 while(b_frames && !s->input_picture[b_frames]) b_frames--;
1280             }else if(s->avctx->b_frame_strategy==1){
1281                 for(i=1; i<s->max_b_frames+1; i++){
1282                     if(s->input_picture[i] && s->input_picture[i]->b_frame_score==0){
1283                         s->input_picture[i]->b_frame_score=
1284                             get_intra_count(s, s->input_picture[i  ]->f.data[0],
1285                                                s->input_picture[i-1]->f.data[0], s->linesize) + 1;
1286                     }
1287                 }
1288                 for(i=0; i<s->max_b_frames+1; i++){
1289                     if(s->input_picture[i]==NULL || s->input_picture[i]->b_frame_score - 1 > s->mb_num/s->avctx->b_sensitivity) break;
1290                 }
1291
1292                 b_frames= FFMAX(0, i-1);
1293
1294                 /* reset scores */
1295                 for(i=0; i<b_frames+1; i++){
1296                     s->input_picture[i]->b_frame_score=0;
1297                 }
1298             }else if(s->avctx->b_frame_strategy==2){
1299                 b_frames= estimate_best_b_count(s);
1300             }else{
1301                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1302                 b_frames=0;
1303             }
1304
1305             emms_c();
1306 //static int b_count=0;
1307 //b_count+= b_frames;
1308 //av_log(s->avctx, AV_LOG_DEBUG, "b_frames: %d\n", b_count);
1309
1310             for(i= b_frames - 1; i>=0; i--){
1311                 int type = s->input_picture[i]->f.pict_type;
1312                 if(type && type != AV_PICTURE_TYPE_B)
1313                     b_frames= i;
1314             }
1315             if (s->input_picture[b_frames]->f.pict_type == AV_PICTURE_TYPE_B && b_frames == s->max_b_frames){
1316                 av_log(s->avctx, AV_LOG_ERROR, "warning, too many b frames in a row\n");
1317             }
1318
1319             if(s->picture_in_gop_number + b_frames >= s->gop_size){
1320               if((s->flags2 & CODEC_FLAG2_STRICT_GOP) && s->gop_size > s->picture_in_gop_number){
1321                     b_frames= s->gop_size - s->picture_in_gop_number - 1;
1322               }else{
1323                 if(s->flags & CODEC_FLAG_CLOSED_GOP)
1324                     b_frames=0;
1325                 s->input_picture[b_frames]->f.pict_type = AV_PICTURE_TYPE_I;
1326               }
1327             }
1328
1329             if(   (s->flags & CODEC_FLAG_CLOSED_GOP)
1330                && b_frames
1331                && s->input_picture[b_frames]->f.pict_type== AV_PICTURE_TYPE_I)
1332                 b_frames--;
1333
1334             s->reordered_input_picture[0]= s->input_picture[b_frames];
1335             if (s->reordered_input_picture[0]->f.pict_type != AV_PICTURE_TYPE_I)
1336                 s->reordered_input_picture[0]->f.pict_type = AV_PICTURE_TYPE_P;
1337             s->reordered_input_picture[0]->f.coded_picture_number = s->coded_picture_number++;
1338             for(i=0; i<b_frames; i++){
1339                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1340                 s->reordered_input_picture[i + 1]->f.pict_type = AV_PICTURE_TYPE_B;
1341                 s->reordered_input_picture[i + 1]->f.coded_picture_number = s->coded_picture_number++;
1342             }
1343         }
1344     }
1345 no_output_pic:
1346     if(s->reordered_input_picture[0]){
1347         s->reordered_input_picture[0]->f.reference = s->reordered_input_picture[0]->f.pict_type!=AV_PICTURE_TYPE_B ? 3 : 0;
1348
1349         ff_copy_picture(&s->new_picture, s->reordered_input_picture[0]);
1350
1351         if (s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_SHARED || s->avctx->rc_buffer_size) {
1352             // input is a shared pix, so we can't modifiy it -> alloc a new one & ensure that the shared one is reuseable
1353
1354             Picture *pic;
1355             int i= ff_find_unused_picture(s, 0);
1356             if (i < 0)
1357                 return i;
1358             pic = &s->picture[i];
1359
1360             pic->f.reference = s->reordered_input_picture[0]->f.reference;
1361             if(ff_alloc_picture(s, pic, 0) < 0){
1362                 return -1;
1363             }
1364
1365             /* mark us unused / free shared pic */
1366             if (s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_INTERNAL)
1367                 s->avctx->release_buffer(s->avctx, (AVFrame*)s->reordered_input_picture[0]);
1368             for(i=0; i<4; i++)
1369                 s->reordered_input_picture[0]->f.data[i] = NULL;
1370             s->reordered_input_picture[0]->f.type = 0;
1371
1372             copy_picture_attributes(s, (AVFrame*)pic, (AVFrame*)s->reordered_input_picture[0]);
1373
1374             s->current_picture_ptr= pic;
1375         }else{
1376             // input is not a shared pix -> reuse buffer for current_pix
1377
1378             assert(   s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_USER
1379                    || s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_INTERNAL);
1380
1381             s->current_picture_ptr= s->reordered_input_picture[0];
1382             for(i=0; i<4; i++){
1383                 s->new_picture.f.data[i] += INPLACE_OFFSET;
1384             }
1385         }
1386         ff_copy_picture(&s->current_picture, s->current_picture_ptr);
1387
1388         s->picture_number = s->new_picture.f.display_picture_number;
1389 //printf("dpn:%d\n", s->picture_number);
1390     }else{
1391        memset(&s->new_picture, 0, sizeof(Picture));
1392     }
1393     return 0;
1394 }
1395
1396 int MPV_encode_picture(AVCodecContext *avctx,
1397                        unsigned char *buf, int buf_size, void *data)
1398 {
1399     MpegEncContext *s = avctx->priv_data;
1400     AVFrame *pic_arg = data;
1401     int i, stuffing_count, context_count = avctx->thread_count;
1402
1403     for(i=0; i<context_count; i++){
1404         int start_y= s->thread_context[i]->start_mb_y;
1405         int   end_y= s->thread_context[i]->  end_mb_y;
1406         int h= s->mb_height;
1407         uint8_t *start= buf + (size_t)(((int64_t) buf_size)*start_y/h);
1408         uint8_t *end  = buf + (size_t)(((int64_t) buf_size)*  end_y/h);
1409
1410         init_put_bits(&s->thread_context[i]->pb, start, end - start);
1411     }
1412
1413     s->picture_in_gop_number++;
1414
1415     if(load_input_picture(s, pic_arg) < 0)
1416         return -1;
1417
1418     if(select_input_picture(s) < 0){
1419         return -1;
1420     }
1421
1422     /* output? */
1423     if (s->new_picture.f.data[0]) {
1424         s->pict_type = s->new_picture.f.pict_type;
1425 //emms_c();
1426 //printf("qs:%f %f %d\n", s->new_picture.quality, s->current_picture.quality, s->qscale);
1427         MPV_frame_start(s, avctx);
1428 vbv_retry:
1429         if (encode_picture(s, s->picture_number) < 0)
1430             return -1;
1431
1432         avctx->header_bits = s->header_bits;
1433         avctx->mv_bits     = s->mv_bits;
1434         avctx->misc_bits   = s->misc_bits;
1435         avctx->i_tex_bits  = s->i_tex_bits;
1436         avctx->p_tex_bits  = s->p_tex_bits;
1437         avctx->i_count     = s->i_count;
1438         avctx->p_count     = s->mb_num - s->i_count - s->skip_count; //FIXME f/b_count in avctx
1439         avctx->skip_count  = s->skip_count;
1440
1441         MPV_frame_end(s);
1442
1443         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1444             ff_mjpeg_encode_picture_trailer(s);
1445
1446         if(avctx->rc_buffer_size){
1447             RateControlContext *rcc= &s->rc_context;
1448             int max_size= rcc->buffer_index * avctx->rc_max_available_vbv_use;
1449
1450             if(put_bits_count(&s->pb) > max_size && s->lambda < s->avctx->lmax){
1451                 s->next_lambda= FFMAX(s->lambda+1, s->lambda*(s->qscale+1) / s->qscale);
1452                 if(s->adaptive_quant){
1453                     int i;
1454                     for(i=0; i<s->mb_height*s->mb_stride; i++)
1455                         s->lambda_table[i]= FFMAX(s->lambda_table[i]+1, s->lambda_table[i]*(s->qscale+1) / s->qscale);
1456                 }
1457                 s->mb_skipped = 0;        //done in MPV_frame_start()
1458                 if(s->pict_type==AV_PICTURE_TYPE_P){ //done in encode_picture() so we must undo it
1459                     if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
1460                         s->no_rounding ^= 1;
1461                 }
1462                 if(s->pict_type!=AV_PICTURE_TYPE_B){
1463                     s->time_base= s->last_time_base;
1464                     s->last_non_b_time= s->time - s->pp_time;
1465                 }
1466 //                av_log(NULL, AV_LOG_ERROR, "R:%d ", s->next_lambda);
1467                 for(i=0; i<context_count; i++){
1468                     PutBitContext *pb= &s->thread_context[i]->pb;
1469                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1470                 }
1471                 goto vbv_retry;
1472             }
1473
1474             assert(s->avctx->rc_max_rate);
1475         }
1476
1477         if(s->flags&CODEC_FLAG_PASS1)
1478             ff_write_pass1_stats(s);
1479
1480         for(i=0; i<4; i++){
1481             s->current_picture_ptr->f.error[i]  = s->current_picture.f.error[i];
1482             avctx->error[i]                        += s->current_picture_ptr->f.error[i];
1483         }
1484
1485         if(s->flags&CODEC_FLAG_PASS1)
1486             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits + avctx->i_tex_bits + avctx->p_tex_bits == put_bits_count(&s->pb));
1487         flush_put_bits(&s->pb);
1488         s->frame_bits  = put_bits_count(&s->pb);
1489
1490         stuffing_count= ff_vbv_update(s, s->frame_bits);
1491         if(stuffing_count){
1492             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < stuffing_count + 50){
1493                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1494                 return -1;
1495             }
1496
1497             switch(s->codec_id){
1498             case CODEC_ID_MPEG1VIDEO:
1499             case CODEC_ID_MPEG2VIDEO:
1500                 while(stuffing_count--){
1501                     put_bits(&s->pb, 8, 0);
1502                 }
1503             break;
1504             case CODEC_ID_MPEG4:
1505                 put_bits(&s->pb, 16, 0);
1506                 put_bits(&s->pb, 16, 0x1C3);
1507                 stuffing_count -= 4;
1508                 while(stuffing_count--){
1509                     put_bits(&s->pb, 8, 0xFF);
1510                 }
1511             break;
1512             default:
1513                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1514             }
1515             flush_put_bits(&s->pb);
1516             s->frame_bits  = put_bits_count(&s->pb);
1517         }
1518
1519         /* update mpeg1/2 vbv_delay for CBR */
1520         if(s->avctx->rc_max_rate && s->avctx->rc_min_rate == s->avctx->rc_max_rate && s->out_format == FMT_MPEG1
1521            && 90000LL * (avctx->rc_buffer_size-1) <= s->avctx->rc_max_rate*0xFFFFLL){
1522             int vbv_delay, min_delay;
1523             double inbits = s->avctx->rc_max_rate*av_q2d(s->avctx->time_base);
1524             int    minbits= s->frame_bits - 8*(s->vbv_delay_ptr - s->pb.buf - 1);
1525             double bits   = s->rc_context.buffer_index + minbits - inbits;
1526
1527             if(bits<0)
1528                 av_log(s->avctx, AV_LOG_ERROR, "Internal error, negative bits\n");
1529
1530             assert(s->repeat_first_field==0);
1531
1532             vbv_delay=     bits * 90000                               / s->avctx->rc_max_rate;
1533             min_delay= (minbits * 90000LL + s->avctx->rc_max_rate - 1)/ s->avctx->rc_max_rate;
1534
1535             vbv_delay= FFMAX(vbv_delay, min_delay);
1536
1537             assert(vbv_delay < 0xFFFF);
1538
1539             s->vbv_delay_ptr[0] &= 0xF8;
1540             s->vbv_delay_ptr[0] |= vbv_delay>>13;
1541             s->vbv_delay_ptr[1]  = vbv_delay>>5;
1542             s->vbv_delay_ptr[2] &= 0x07;
1543             s->vbv_delay_ptr[2] |= vbv_delay<<3;
1544             avctx->vbv_delay = vbv_delay*300;
1545         }
1546         s->total_bits += s->frame_bits;
1547         avctx->frame_bits  = s->frame_bits;
1548     }else{
1549         assert((put_bits_ptr(&s->pb) == s->pb.buf));
1550         s->frame_bits=0;
1551     }
1552     assert((s->frame_bits&7)==0);
1553
1554     return s->frame_bits/8;
1555 }
1556
1557 static inline void dct_single_coeff_elimination(MpegEncContext *s, int n, int threshold)
1558 {
1559     static const char tab[64]=
1560         {3,2,2,1,1,1,1,1,
1561          1,1,1,1,1,1,1,1,
1562          1,1,1,1,1,1,1,1,
1563          0,0,0,0,0,0,0,0,
1564          0,0,0,0,0,0,0,0,
1565          0,0,0,0,0,0,0,0,
1566          0,0,0,0,0,0,0,0,
1567          0,0,0,0,0,0,0,0};
1568     int score=0;
1569     int run=0;
1570     int i;
1571     DCTELEM *block= s->block[n];
1572     const int last_index= s->block_last_index[n];
1573     int skip_dc;
1574
1575     if(threshold<0){
1576         skip_dc=0;
1577         threshold= -threshold;
1578     }else
1579         skip_dc=1;
1580
1581     /* Are all we could set to zero already zero? */
1582     if(last_index<=skip_dc - 1) return;
1583
1584     for(i=0; i<=last_index; i++){
1585         const int j = s->intra_scantable.permutated[i];
1586         const int level = FFABS(block[j]);
1587         if(level==1){
1588             if(skip_dc && i==0) continue;
1589             score+= tab[run];
1590             run=0;
1591         }else if(level>1){
1592             return;
1593         }else{
1594             run++;
1595         }
1596     }
1597     if(score >= threshold) return;
1598     for(i=skip_dc; i<=last_index; i++){
1599         const int j = s->intra_scantable.permutated[i];
1600         block[j]=0;
1601     }
1602     if(block[0]) s->block_last_index[n]= 0;
1603     else         s->block_last_index[n]= -1;
1604 }
1605
1606 static inline void clip_coeffs(MpegEncContext *s, DCTELEM *block, int last_index)
1607 {
1608     int i;
1609     const int maxlevel= s->max_qcoeff;
1610     const int minlevel= s->min_qcoeff;
1611     int overflow=0;
1612
1613     if(s->mb_intra){
1614         i=1; //skip clipping of intra dc
1615     }else
1616         i=0;
1617
1618     for(;i<=last_index; i++){
1619         const int j= s->intra_scantable.permutated[i];
1620         int level = block[j];
1621
1622         if     (level>maxlevel){
1623             level=maxlevel;
1624             overflow++;
1625         }else if(level<minlevel){
1626             level=minlevel;
1627             overflow++;
1628         }
1629
1630         block[j]= level;
1631     }
1632
1633     if(overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1634         av_log(s->avctx, AV_LOG_INFO, "warning, clipping %d dct coefficients to %d..%d\n", overflow, minlevel, maxlevel);
1635 }
1636
1637 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride){
1638     int x, y;
1639 //FIXME optimize
1640     for(y=0; y<8; y++){
1641         for(x=0; x<8; x++){
1642             int x2, y2;
1643             int sum=0;
1644             int sqr=0;
1645             int count=0;
1646
1647             for(y2= FFMAX(y-1, 0); y2 < FFMIN(8, y+2); y2++){
1648                 for(x2= FFMAX(x-1, 0); x2 < FFMIN(8, x+2); x2++){
1649                     int v= ptr[x2 + y2*stride];
1650                     sum += v;
1651                     sqr += v*v;
1652                     count++;
1653                 }
1654             }
1655             weight[x + 8*y]= (36*ff_sqrt(count*sqr - sum*sum)) / count;
1656         }
1657     }
1658 }
1659
1660 static av_always_inline void encode_mb_internal(MpegEncContext *s, int motion_x, int motion_y, int mb_block_height, int mb_block_count)
1661 {
1662     int16_t weight[8][64];
1663     DCTELEM orig[8][64];
1664     const int mb_x= s->mb_x;
1665     const int mb_y= s->mb_y;
1666     int i;
1667     int skip_dct[8];
1668     int dct_offset   = s->linesize*8; //default for progressive frames
1669     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1670     int wrap_y, wrap_c;
1671
1672     for(i=0; i<mb_block_count; i++) skip_dct[i]=s->skipdct;
1673
1674     if(s->adaptive_quant){
1675         const int last_qp= s->qscale;
1676         const int mb_xy= mb_x + mb_y*s->mb_stride;
1677
1678         s->lambda= s->lambda_table[mb_xy];
1679         update_qscale(s);
1680
1681         if(!(s->flags&CODEC_FLAG_QP_RD)){
1682             s->qscale = s->current_picture_ptr->f.qscale_table[mb_xy];
1683             s->dquant= s->qscale - last_qp;
1684
1685             if(s->out_format==FMT_H263){
1686                 s->dquant= av_clip(s->dquant, -2, 2);
1687
1688                 if(s->codec_id==CODEC_ID_MPEG4){
1689                     if(!s->mb_intra){
1690                         if(s->pict_type == AV_PICTURE_TYPE_B){
1691                             if(s->dquant&1 || s->mv_dir&MV_DIRECT)
1692                                 s->dquant= 0;
1693                         }
1694                         if(s->mv_type==MV_TYPE_8X8)
1695                             s->dquant=0;
1696                     }
1697                 }
1698             }
1699         }
1700         ff_set_qscale(s, last_qp + s->dquant);
1701     }else if(s->flags&CODEC_FLAG_QP_RD)
1702         ff_set_qscale(s, s->qscale + s->dquant);
1703
1704     wrap_y = s->linesize;
1705     wrap_c = s->uvlinesize;
1706     ptr_y  = s->new_picture.f.data[0] + (mb_y * 16 * wrap_y) + mb_x * 16;
1707     ptr_cb = s->new_picture.f.data[1] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1708     ptr_cr = s->new_picture.f.data[2] + (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1709
1710     if((mb_x*16+16 > s->width || mb_y*16+16 > s->height) && s->codec_id != CODEC_ID_AMV){
1711         uint8_t *ebuf= s->edge_emu_buffer + 32;
1712         s->dsp.emulated_edge_mc(ebuf            , ptr_y , wrap_y,16,16,mb_x*16,mb_y*16, s->width   , s->height);
1713         ptr_y= ebuf;
1714         s->dsp.emulated_edge_mc(ebuf+18*wrap_y  , ptr_cb, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1715         ptr_cb= ebuf+18*wrap_y;
1716         s->dsp.emulated_edge_mc(ebuf+18*wrap_y+8, ptr_cr, wrap_c, 8, mb_block_height, mb_x*8, mb_y*8, s->width>>1, s->height>>1);
1717         ptr_cr= ebuf+18*wrap_y+8;
1718     }
1719
1720     if (s->mb_intra) {
1721         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1722             int progressive_score, interlaced_score;
1723
1724             s->interlaced_dct=0;
1725             progressive_score= s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y, 8)
1726                               +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y*8, NULL, wrap_y, 8) - 400;
1727
1728             if(progressive_score > 0){
1729                 interlaced_score = s->dsp.ildct_cmp[4](s, ptr_y           , NULL, wrap_y*2, 8)
1730                                   +s->dsp.ildct_cmp[4](s, ptr_y + wrap_y  , NULL, wrap_y*2, 8);
1731                 if(progressive_score > interlaced_score){
1732                     s->interlaced_dct=1;
1733
1734                     dct_offset= wrap_y;
1735                     wrap_y<<=1;
1736                     if (s->chroma_format == CHROMA_422)
1737                         wrap_c<<=1;
1738                 }
1739             }
1740         }
1741
1742         s->dsp.get_pixels(s->block[0], ptr_y                 , wrap_y);
1743         s->dsp.get_pixels(s->block[1], ptr_y              + 8, wrap_y);
1744         s->dsp.get_pixels(s->block[2], ptr_y + dct_offset    , wrap_y);
1745         s->dsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
1746
1747         if(s->flags&CODEC_FLAG_GRAY){
1748             skip_dct[4]= 1;
1749             skip_dct[5]= 1;
1750         }else{
1751             s->dsp.get_pixels(s->block[4], ptr_cb, wrap_c);
1752             s->dsp.get_pixels(s->block[5], ptr_cr, wrap_c);
1753             if(!s->chroma_y_shift){ /* 422 */
1754                 s->dsp.get_pixels(s->block[6], ptr_cb + (dct_offset>>1), wrap_c);
1755                 s->dsp.get_pixels(s->block[7], ptr_cr + (dct_offset>>1), wrap_c);
1756             }
1757         }
1758     }else{
1759         op_pixels_func (*op_pix)[4];
1760         qpel_mc_func (*op_qpix)[16];
1761         uint8_t *dest_y, *dest_cb, *dest_cr;
1762
1763         dest_y  = s->dest[0];
1764         dest_cb = s->dest[1];
1765         dest_cr = s->dest[2];
1766
1767         if ((!s->no_rounding) || s->pict_type==AV_PICTURE_TYPE_B){
1768             op_pix = s->dsp.put_pixels_tab;
1769             op_qpix= s->dsp.put_qpel_pixels_tab;
1770         }else{
1771             op_pix = s->dsp.put_no_rnd_pixels_tab;
1772             op_qpix= s->dsp.put_no_rnd_qpel_pixels_tab;
1773         }
1774
1775         if (s->mv_dir & MV_DIR_FORWARD) {
1776             MPV_motion(s, dest_y, dest_cb, dest_cr, 0, s->last_picture.f.data, op_pix, op_qpix);
1777             op_pix = s->dsp.avg_pixels_tab;
1778             op_qpix= s->dsp.avg_qpel_pixels_tab;
1779         }
1780         if (s->mv_dir & MV_DIR_BACKWARD) {
1781             MPV_motion(s, dest_y, dest_cb, dest_cr, 1, s->next_picture.f.data, op_pix, op_qpix);
1782         }
1783
1784         if(s->flags&CODEC_FLAG_INTERLACED_DCT){
1785             int progressive_score, interlaced_score;
1786
1787             s->interlaced_dct=0;
1788             progressive_score= s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y, 8)
1789                               +s->dsp.ildct_cmp[0](s, dest_y + wrap_y*8, ptr_y + wrap_y*8, wrap_y, 8) - 400;
1790
1791             if(s->avctx->ildct_cmp == FF_CMP_VSSE) progressive_score -= 400;
1792
1793             if(progressive_score>0){
1794                 interlaced_score = s->dsp.ildct_cmp[0](s, dest_y           , ptr_y           , wrap_y*2, 8)
1795                                   +s->dsp.ildct_cmp[0](s, dest_y + wrap_y  , ptr_y + wrap_y  , wrap_y*2, 8);
1796
1797                 if(progressive_score > interlaced_score){
1798                     s->interlaced_dct=1;
1799
1800                     dct_offset= wrap_y;
1801                     wrap_y<<=1;
1802                     if (s->chroma_format == CHROMA_422)
1803                         wrap_c<<=1;
1804                 }
1805             }
1806         }
1807
1808         s->dsp.diff_pixels(s->block[0], ptr_y                 , dest_y                 , wrap_y);
1809         s->dsp.diff_pixels(s->block[1], ptr_y              + 8, dest_y              + 8, wrap_y);
1810         s->dsp.diff_pixels(s->block[2], ptr_y + dct_offset    , dest_y + dct_offset    , wrap_y);
1811         s->dsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8, dest_y + dct_offset + 8, wrap_y);
1812
1813         if(s->flags&CODEC_FLAG_GRAY){
1814             skip_dct[4]= 1;
1815             skip_dct[5]= 1;
1816         }else{
1817             s->dsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
1818             s->dsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
1819             if(!s->chroma_y_shift){ /* 422 */
1820                 s->dsp.diff_pixels(s->block[6], ptr_cb + (dct_offset>>1), dest_cb + (dct_offset>>1), wrap_c);
1821                 s->dsp.diff_pixels(s->block[7], ptr_cr + (dct_offset>>1), dest_cr + (dct_offset>>1), wrap_c);
1822             }
1823         }
1824         /* pre quantization */
1825         if(s->current_picture.mc_mb_var[s->mb_stride*mb_y+ mb_x]<2*s->qscale*s->qscale){
1826             //FIXME optimize
1827             if(s->dsp.sad[1](NULL, ptr_y               , dest_y               , wrap_y, 8) < 20*s->qscale) skip_dct[0]= 1;
1828             if(s->dsp.sad[1](NULL, ptr_y            + 8, dest_y            + 8, wrap_y, 8) < 20*s->qscale) skip_dct[1]= 1;
1829             if(s->dsp.sad[1](NULL, ptr_y +dct_offset   , dest_y +dct_offset   , wrap_y, 8) < 20*s->qscale) skip_dct[2]= 1;
1830             if(s->dsp.sad[1](NULL, ptr_y +dct_offset+ 8, dest_y +dct_offset+ 8, wrap_y, 8) < 20*s->qscale) skip_dct[3]= 1;
1831             if(s->dsp.sad[1](NULL, ptr_cb              , dest_cb              , wrap_c, 8) < 20*s->qscale) skip_dct[4]= 1;
1832             if(s->dsp.sad[1](NULL, ptr_cr              , dest_cr              , wrap_c, 8) < 20*s->qscale) skip_dct[5]= 1;
1833             if(!s->chroma_y_shift){ /* 422 */
1834                 if(s->dsp.sad[1](NULL, ptr_cb +(dct_offset>>1), dest_cb +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[6]= 1;
1835                 if(s->dsp.sad[1](NULL, ptr_cr +(dct_offset>>1), dest_cr +(dct_offset>>1), wrap_c, 8) < 20*s->qscale) skip_dct[7]= 1;
1836             }
1837         }
1838     }
1839
1840     if(s->avctx->quantizer_noise_shaping){
1841         if(!skip_dct[0]) get_visual_weight(weight[0], ptr_y                 , wrap_y);
1842         if(!skip_dct[1]) get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
1843         if(!skip_dct[2]) get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
1844         if(!skip_dct[3]) get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
1845         if(!skip_dct[4]) get_visual_weight(weight[4], ptr_cb                , wrap_c);
1846         if(!skip_dct[5]) get_visual_weight(weight[5], ptr_cr                , wrap_c);
1847         if(!s->chroma_y_shift){ /* 422 */
1848             if(!skip_dct[6]) get_visual_weight(weight[6], ptr_cb + (dct_offset>>1), wrap_c);
1849             if(!skip_dct[7]) get_visual_weight(weight[7], ptr_cr + (dct_offset>>1), wrap_c);
1850         }
1851         memcpy(orig[0], s->block[0], sizeof(DCTELEM)*64*mb_block_count);
1852     }
1853
1854     /* DCT & quantize */
1855     assert(s->out_format!=FMT_MJPEG || s->qscale==8);
1856     {
1857         for(i=0;i<mb_block_count;i++) {
1858             if(!skip_dct[i]){
1859                 int overflow;
1860                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
1861             // FIXME we could decide to change to quantizer instead of clipping
1862             // JS: I don't think that would be a good idea it could lower quality instead
1863             //     of improve it. Just INTRADC clipping deserves changes in quantizer
1864                 if (overflow) clip_coeffs(s, s->block[i], s->block_last_index[i]);
1865             }else
1866                 s->block_last_index[i]= -1;
1867         }
1868         if(s->avctx->quantizer_noise_shaping){
1869             for(i=0;i<mb_block_count;i++) {
1870                 if(!skip_dct[i]){
1871                     s->block_last_index[i] = dct_quantize_refine(s, s->block[i], weight[i], orig[i], i, s->qscale);
1872                 }
1873             }
1874         }
1875
1876         if(s->luma_elim_threshold && !s->mb_intra)
1877             for(i=0; i<4; i++)
1878                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
1879         if(s->chroma_elim_threshold && !s->mb_intra)
1880             for(i=4; i<mb_block_count; i++)
1881                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
1882
1883         if(s->flags & CODEC_FLAG_CBP_RD){
1884             for(i=0;i<mb_block_count;i++) {
1885                 if(s->block_last_index[i] == -1)
1886                     s->coded_score[i]= INT_MAX/256;
1887             }
1888         }
1889     }
1890
1891     if((s->flags&CODEC_FLAG_GRAY) && s->mb_intra){
1892         s->block_last_index[4]=
1893         s->block_last_index[5]= 0;
1894         s->block[4][0]=
1895         s->block[5][0]= (1024 + s->c_dc_scale/2)/ s->c_dc_scale;
1896     }
1897
1898     //non c quantize code returns incorrect block_last_index FIXME
1899     if(s->alternate_scan && s->dct_quantize != dct_quantize_c){
1900         for(i=0; i<mb_block_count; i++){
1901             int j;
1902             if(s->block_last_index[i]>0){
1903                 for(j=63; j>0; j--){
1904                     if(s->block[i][ s->intra_scantable.permutated[j] ]) break;
1905                 }
1906                 s->block_last_index[i]= j;
1907             }
1908         }
1909     }
1910
1911     /* huffman encode */
1912     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
1913     case CODEC_ID_MPEG1VIDEO:
1914     case CODEC_ID_MPEG2VIDEO:
1915         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
1916             mpeg1_encode_mb(s, s->block, motion_x, motion_y);
1917         break;
1918     case CODEC_ID_MPEG4:
1919         if (CONFIG_MPEG4_ENCODER)
1920             mpeg4_encode_mb(s, s->block, motion_x, motion_y);
1921         break;
1922     case CODEC_ID_MSMPEG4V2:
1923     case CODEC_ID_MSMPEG4V3:
1924     case CODEC_ID_WMV1:
1925         if (CONFIG_MSMPEG4_ENCODER)
1926             msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
1927         break;
1928     case CODEC_ID_WMV2:
1929         if (CONFIG_WMV2_ENCODER)
1930             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
1931         break;
1932     case CODEC_ID_H261:
1933         if (CONFIG_H261_ENCODER)
1934             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
1935         break;
1936     case CODEC_ID_H263:
1937     case CODEC_ID_H263P:
1938     case CODEC_ID_FLV1:
1939     case CODEC_ID_RV10:
1940     case CODEC_ID_RV20:
1941         if (CONFIG_H263_ENCODER)
1942             h263_encode_mb(s, s->block, motion_x, motion_y);
1943         break;
1944     case CODEC_ID_MJPEG:
1945     case CODEC_ID_AMV:
1946         if (CONFIG_MJPEG_ENCODER)
1947             ff_mjpeg_encode_mb(s, s->block);
1948         break;
1949     default:
1950         assert(0);
1951     }
1952 }
1953
1954 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
1955 {
1956     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
1957     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
1958 }
1959
1960 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
1961     int i;
1962
1963     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
1964
1965     /* mpeg1 */
1966     d->mb_skip_run= s->mb_skip_run;
1967     for(i=0; i<3; i++)
1968         d->last_dc[i]= s->last_dc[i];
1969
1970     /* statistics */
1971     d->mv_bits= s->mv_bits;
1972     d->i_tex_bits= s->i_tex_bits;
1973     d->p_tex_bits= s->p_tex_bits;
1974     d->i_count= s->i_count;
1975     d->f_count= s->f_count;
1976     d->b_count= s->b_count;
1977     d->skip_count= s->skip_count;
1978     d->misc_bits= s->misc_bits;
1979     d->last_bits= 0;
1980
1981     d->mb_skipped= 0;
1982     d->qscale= s->qscale;
1983     d->dquant= s->dquant;
1984
1985     d->esc3_level_length= s->esc3_level_length;
1986 }
1987
1988 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
1989     int i;
1990
1991     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
1992     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
1993
1994     /* mpeg1 */
1995     d->mb_skip_run= s->mb_skip_run;
1996     for(i=0; i<3; i++)
1997         d->last_dc[i]= s->last_dc[i];
1998
1999     /* statistics */
2000     d->mv_bits= s->mv_bits;
2001     d->i_tex_bits= s->i_tex_bits;
2002     d->p_tex_bits= s->p_tex_bits;
2003     d->i_count= s->i_count;
2004     d->f_count= s->f_count;
2005     d->b_count= s->b_count;
2006     d->skip_count= s->skip_count;
2007     d->misc_bits= s->misc_bits;
2008
2009     d->mb_intra= s->mb_intra;
2010     d->mb_skipped= s->mb_skipped;
2011     d->mv_type= s->mv_type;
2012     d->mv_dir= s->mv_dir;
2013     d->pb= s->pb;
2014     if(s->data_partitioning){
2015         d->pb2= s->pb2;
2016         d->tex_pb= s->tex_pb;
2017     }
2018     d->block= s->block;
2019     for(i=0; i<8; i++)
2020         d->block_last_index[i]= s->block_last_index[i];
2021     d->interlaced_dct= s->interlaced_dct;
2022     d->qscale= s->qscale;
2023
2024     d->esc3_level_length= s->esc3_level_length;
2025 }
2026
2027 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2028                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2029                            int *dmin, int *next_block, int motion_x, int motion_y)
2030 {
2031     int score;
2032     uint8_t *dest_backup[3];
2033
2034     copy_context_before_encode(s, backup, type);
2035
2036     s->block= s->blocks[*next_block];
2037     s->pb= pb[*next_block];
2038     if(s->data_partitioning){
2039         s->pb2   = pb2   [*next_block];
2040         s->tex_pb= tex_pb[*next_block];
2041     }
2042
2043     if(*next_block){
2044         memcpy(dest_backup, s->dest, sizeof(s->dest));
2045         s->dest[0] = s->rd_scratchpad;
2046         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
2047         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
2048         assert(s->linesize >= 32); //FIXME
2049     }
2050
2051     encode_mb(s, motion_x, motion_y);
2052
2053     score= put_bits_count(&s->pb);
2054     if(s->data_partitioning){
2055         score+= put_bits_count(&s->pb2);
2056         score+= put_bits_count(&s->tex_pb);
2057     }
2058
2059     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2060         MPV_decode_mb(s, s->block);
2061
2062         score *= s->lambda2;
2063         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2064     }
2065
2066     if(*next_block){
2067         memcpy(s->dest, dest_backup, sizeof(s->dest));
2068     }
2069
2070     if(score<*dmin){
2071         *dmin= score;
2072         *next_block^=1;
2073
2074         copy_context_after_encode(best, s, type);
2075     }
2076 }
2077
2078 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2079     uint32_t *sq = ff_squareTbl + 256;
2080     int acc=0;
2081     int x,y;
2082
2083     if(w==16 && h==16)
2084         return s->dsp.sse[0](NULL, src1, src2, stride, 16);
2085     else if(w==8 && h==8)
2086         return s->dsp.sse[1](NULL, src1, src2, stride, 8);
2087
2088     for(y=0; y<h; y++){
2089         for(x=0; x<w; x++){
2090             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2091         }
2092     }
2093
2094     assert(acc>=0);
2095
2096     return acc;
2097 }
2098
2099 static int sse_mb(MpegEncContext *s){
2100     int w= 16;
2101     int h= 16;
2102
2103     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2104     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2105
2106     if(w==16 && h==16)
2107       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2108         return  s->dsp.nsse[0](s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2109                +s->dsp.nsse[1](s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2110                +s->dsp.nsse[1](s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2111       }else{
2112         return  s->dsp.sse[0](NULL, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2113                +s->dsp.sse[1](NULL, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2114                +s->dsp.sse[1](NULL, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2115       }
2116     else
2117         return  sse(s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2118                +sse(s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2119                +sse(s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2120 }
2121
2122 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2123     MpegEncContext *s= *(void**)arg;
2124
2125
2126     s->me.pre_pass=1;
2127     s->me.dia_size= s->avctx->pre_dia_size;
2128     s->first_slice_line=1;
2129     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2130         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2131             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2132         }
2133         s->first_slice_line=0;
2134     }
2135
2136     s->me.pre_pass=0;
2137
2138     return 0;
2139 }
2140
2141 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2142     MpegEncContext *s= *(void**)arg;
2143
2144     ff_check_alignment();
2145
2146     s->me.dia_size= s->avctx->dia_size;
2147     s->first_slice_line=1;
2148     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2149         s->mb_x=0; //for block init below
2150         ff_init_block_index(s);
2151         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2152             s->block_index[0]+=2;
2153             s->block_index[1]+=2;
2154             s->block_index[2]+=2;
2155             s->block_index[3]+=2;
2156
2157             /* compute motion vector & mb_type and store in context */
2158             if(s->pict_type==AV_PICTURE_TYPE_B)
2159                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2160             else
2161                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2162         }
2163         s->first_slice_line=0;
2164     }
2165     return 0;
2166 }
2167
2168 static int mb_var_thread(AVCodecContext *c, void *arg){
2169     MpegEncContext *s= *(void**)arg;
2170     int mb_x, mb_y;
2171
2172     ff_check_alignment();
2173
2174     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2175         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2176             int xx = mb_x * 16;
2177             int yy = mb_y * 16;
2178             uint8_t *pix = s->new_picture.f.data[0] + (yy * s->linesize) + xx;
2179             int varc;
2180             int sum = s->dsp.pix_sum(pix, s->linesize);
2181
2182             varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)sum*sum)>>8) + 500 + 128)>>8;
2183
2184             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2185             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2186             s->me.mb_var_sum_temp    += varc;
2187         }
2188     }
2189     return 0;
2190 }
2191
2192 static void write_slice_end(MpegEncContext *s){
2193     if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4){
2194         if(s->partitioned_frame){
2195             ff_mpeg4_merge_partitions(s);
2196         }
2197
2198         ff_mpeg4_stuffing(&s->pb);
2199     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2200         ff_mjpeg_encode_stuffing(&s->pb);
2201     }
2202
2203     avpriv_align_put_bits(&s->pb);
2204     flush_put_bits(&s->pb);
2205
2206     if((s->flags&CODEC_FLAG_PASS1) && !s->partitioned_frame)
2207         s->misc_bits+= get_bits_diff(s);
2208 }
2209
2210 static int encode_thread(AVCodecContext *c, void *arg){
2211     MpegEncContext *s= *(void**)arg;
2212     int mb_x, mb_y, pdif = 0;
2213     int chr_h= 16>>s->chroma_y_shift;
2214     int i, j;
2215     MpegEncContext best_s, backup_s;
2216     uint8_t bit_buf[2][MAX_MB_BYTES];
2217     uint8_t bit_buf2[2][MAX_MB_BYTES];
2218     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2219     PutBitContext pb[2], pb2[2], tex_pb[2];
2220 //printf("%d->%d\n", s->resync_mb_y, s->end_mb_y);
2221
2222     ff_check_alignment();
2223
2224     for(i=0; i<2; i++){
2225         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2226         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2227         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2228     }
2229
2230     s->last_bits= put_bits_count(&s->pb);
2231     s->mv_bits=0;
2232     s->misc_bits=0;
2233     s->i_tex_bits=0;
2234     s->p_tex_bits=0;
2235     s->i_count=0;
2236     s->f_count=0;
2237     s->b_count=0;
2238     s->skip_count=0;
2239
2240     for(i=0; i<3; i++){
2241         /* init last dc values */
2242         /* note: quant matrix value (8) is implied here */
2243         s->last_dc[i] = 128 << s->intra_dc_precision;
2244
2245         s->current_picture.f.error[i] = 0;
2246     }
2247     if(s->codec_id==CODEC_ID_AMV){
2248         s->last_dc[0] = 128*8/13;
2249         s->last_dc[1] = 128*8/14;
2250         s->last_dc[2] = 128*8/14;
2251     }
2252     s->mb_skip_run = 0;
2253     memset(s->last_mv, 0, sizeof(s->last_mv));
2254
2255     s->last_mv_dir = 0;
2256
2257     switch(s->codec_id){
2258     case CODEC_ID_H263:
2259     case CODEC_ID_H263P:
2260     case CODEC_ID_FLV1:
2261         if (CONFIG_H263_ENCODER)
2262             s->gob_index = ff_h263_get_gob_height(s);
2263         break;
2264     case CODEC_ID_MPEG4:
2265         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2266             ff_mpeg4_init_partitions(s);
2267         break;
2268     }
2269
2270     s->resync_mb_x=0;
2271     s->resync_mb_y=0;
2272     s->first_slice_line = 1;
2273     s->ptr_lastgob = s->pb.buf;
2274     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2275 //    printf("row %d at %X\n", s->mb_y, (int)s);
2276         s->mb_x=0;
2277         s->mb_y= mb_y;
2278
2279         ff_set_qscale(s, s->qscale);
2280         ff_init_block_index(s);
2281
2282         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2283             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2284             int mb_type= s->mb_type[xy];
2285 //            int d;
2286             int dmin= INT_MAX;
2287             int dir;
2288
2289             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2290                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2291                 return -1;
2292             }
2293             if(s->data_partitioning){
2294                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2295                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2296                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2297                     return -1;
2298                 }
2299             }
2300
2301             s->mb_x = mb_x;
2302             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2303             ff_update_block_index(s);
2304
2305             if(CONFIG_H261_ENCODER && s->codec_id == CODEC_ID_H261){
2306                 ff_h261_reorder_mb_index(s);
2307                 xy= s->mb_y*s->mb_stride + s->mb_x;
2308                 mb_type= s->mb_type[xy];
2309             }
2310
2311             /* write gob / video packet header  */
2312             if(s->rtp_mode){
2313                 int current_packet_size, is_gob_start;
2314
2315                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2316
2317                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2318
2319                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2320
2321                 switch(s->codec_id){
2322                 case CODEC_ID_H263:
2323                 case CODEC_ID_H263P:
2324                     if(!s->h263_slice_structured)
2325                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2326                     break;
2327                 case CODEC_ID_MPEG2VIDEO:
2328                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2329                 case CODEC_ID_MPEG1VIDEO:
2330                     if(s->mb_skip_run) is_gob_start=0;
2331                     break;
2332                 }
2333
2334                 if(is_gob_start){
2335                     if(s->start_mb_y != mb_y || mb_x!=0){
2336                         write_slice_end(s);
2337
2338                         if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4 && s->partitioned_frame){
2339                             ff_mpeg4_init_partitions(s);
2340                         }
2341                     }
2342
2343                     assert((put_bits_count(&s->pb)&7) == 0);
2344                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2345
2346                     if(s->avctx->error_rate && s->resync_mb_x + s->resync_mb_y > 0){
2347                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2348                         int d= 100 / s->avctx->error_rate;
2349                         if(r % d == 0){
2350                             current_packet_size=0;
2351                             s->pb.buf_ptr= s->ptr_lastgob;
2352                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2353                         }
2354                     }
2355
2356                     if (s->avctx->rtp_callback){
2357                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2358                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2359                     }
2360
2361                     switch(s->codec_id){
2362                     case CODEC_ID_MPEG4:
2363                         if (CONFIG_MPEG4_ENCODER) {
2364                             ff_mpeg4_encode_video_packet_header(s);
2365                             ff_mpeg4_clean_buffers(s);
2366                         }
2367                     break;
2368                     case CODEC_ID_MPEG1VIDEO:
2369                     case CODEC_ID_MPEG2VIDEO:
2370                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2371                             ff_mpeg1_encode_slice_header(s);
2372                             ff_mpeg1_clean_buffers(s);
2373                         }
2374                     break;
2375                     case CODEC_ID_H263:
2376                     case CODEC_ID_H263P:
2377                         if (CONFIG_H263_ENCODER)
2378                             h263_encode_gob_header(s, mb_y);
2379                     break;
2380                     }
2381
2382                     if(s->flags&CODEC_FLAG_PASS1){
2383                         int bits= put_bits_count(&s->pb);
2384                         s->misc_bits+= bits - s->last_bits;
2385                         s->last_bits= bits;
2386                     }
2387
2388                     s->ptr_lastgob += current_packet_size;
2389                     s->first_slice_line=1;
2390                     s->resync_mb_x=mb_x;
2391                     s->resync_mb_y=mb_y;
2392                 }
2393             }
2394
2395             if(  (s->resync_mb_x   == s->mb_x)
2396                && s->resync_mb_y+1 == s->mb_y){
2397                 s->first_slice_line=0;
2398             }
2399
2400             s->mb_skipped=0;
2401             s->dquant=0; //only for QP_RD
2402
2403             if(mb_type & (mb_type-1) || (s->flags & CODEC_FLAG_QP_RD)){ // more than 1 MB type possible or CODEC_FLAG_QP_RD
2404                 int next_block=0;
2405                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2406
2407                 copy_context_before_encode(&backup_s, s, -1);
2408                 backup_s.pb= s->pb;
2409                 best_s.data_partitioning= s->data_partitioning;
2410                 best_s.partitioned_frame= s->partitioned_frame;
2411                 if(s->data_partitioning){
2412                     backup_s.pb2= s->pb2;
2413                     backup_s.tex_pb= s->tex_pb;
2414                 }
2415
2416                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2417                     s->mv_dir = MV_DIR_FORWARD;
2418                     s->mv_type = MV_TYPE_16X16;
2419                     s->mb_intra= 0;
2420                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2421                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2422                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2423                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2424                 }
2425                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2426                     s->mv_dir = MV_DIR_FORWARD;
2427                     s->mv_type = MV_TYPE_FIELD;
2428                     s->mb_intra= 0;
2429                     for(i=0; i<2; i++){
2430                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2431                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2432                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2433                     }
2434                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2435                                  &dmin, &next_block, 0, 0);
2436                 }
2437                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2438                     s->mv_dir = MV_DIR_FORWARD;
2439                     s->mv_type = MV_TYPE_16X16;
2440                     s->mb_intra= 0;
2441                     s->mv[0][0][0] = 0;
2442                     s->mv[0][0][1] = 0;
2443                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2444                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2445                 }
2446                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2447                     s->mv_dir = MV_DIR_FORWARD;
2448                     s->mv_type = MV_TYPE_8X8;
2449                     s->mb_intra= 0;
2450                     for(i=0; i<4; i++){
2451                         s->mv[0][i][0] = s->current_picture.f.motion_val[0][s->block_index[i]][0];
2452                         s->mv[0][i][1] = s->current_picture.f.motion_val[0][s->block_index[i]][1];
2453                     }
2454                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2455                                  &dmin, &next_block, 0, 0);
2456                 }
2457                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2458                     s->mv_dir = MV_DIR_FORWARD;
2459                     s->mv_type = MV_TYPE_16X16;
2460                     s->mb_intra= 0;
2461                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2462                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2463                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2464                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2465                 }
2466                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2467                     s->mv_dir = MV_DIR_BACKWARD;
2468                     s->mv_type = MV_TYPE_16X16;
2469                     s->mb_intra= 0;
2470                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2471                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2472                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2473                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2474                 }
2475                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2476                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2477                     s->mv_type = MV_TYPE_16X16;
2478                     s->mb_intra= 0;
2479                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2480                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2481                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2482                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2483                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2484                                  &dmin, &next_block, 0, 0);
2485                 }
2486                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2487                     s->mv_dir = MV_DIR_FORWARD;
2488                     s->mv_type = MV_TYPE_FIELD;
2489                     s->mb_intra= 0;
2490                     for(i=0; i<2; i++){
2491                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2492                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2493                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2494                     }
2495                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2496                                  &dmin, &next_block, 0, 0);
2497                 }
2498                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2499                     s->mv_dir = MV_DIR_BACKWARD;
2500                     s->mv_type = MV_TYPE_FIELD;
2501                     s->mb_intra= 0;
2502                     for(i=0; i<2; i++){
2503                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2504                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2505                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2506                     }
2507                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2508                                  &dmin, &next_block, 0, 0);
2509                 }
2510                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2511                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2512                     s->mv_type = MV_TYPE_FIELD;
2513                     s->mb_intra= 0;
2514                     for(dir=0; dir<2; dir++){
2515                         for(i=0; i<2; i++){
2516                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2517                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2518                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2519                         }
2520                     }
2521                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2522                                  &dmin, &next_block, 0, 0);
2523                 }
2524                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2525                     s->mv_dir = 0;
2526                     s->mv_type = MV_TYPE_16X16;
2527                     s->mb_intra= 1;
2528                     s->mv[0][0][0] = 0;
2529                     s->mv[0][0][1] = 0;
2530                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2531                                  &dmin, &next_block, 0, 0);
2532                     if(s->h263_pred || s->h263_aic){
2533                         if(best_s.mb_intra)
2534                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2535                         else
2536                             ff_clean_intra_table_entries(s); //old mode?
2537                     }
2538                 }
2539
2540                 if((s->flags & CODEC_FLAG_QP_RD) && dmin < INT_MAX){
2541                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2542                         const int last_qp= backup_s.qscale;
2543                         int qpi, qp, dc[6];
2544                         DCTELEM ac[6][16];
2545                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2546                         static const int dquant_tab[4]={-1,1,-2,2};
2547
2548                         assert(backup_s.dquant == 0);
2549
2550                         //FIXME intra
2551                         s->mv_dir= best_s.mv_dir;
2552                         s->mv_type = MV_TYPE_16X16;
2553                         s->mb_intra= best_s.mb_intra;
2554                         s->mv[0][0][0] = best_s.mv[0][0][0];
2555                         s->mv[0][0][1] = best_s.mv[0][0][1];
2556                         s->mv[1][0][0] = best_s.mv[1][0][0];
2557                         s->mv[1][0][1] = best_s.mv[1][0][1];
2558
2559                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
2560                         for(; qpi<4; qpi++){
2561                             int dquant= dquant_tab[qpi];
2562                             qp= last_qp + dquant;
2563                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2564                                 continue;
2565                             backup_s.dquant= dquant;
2566                             if(s->mb_intra && s->dc_val[0]){
2567                                 for(i=0; i<6; i++){
2568                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2569                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(DCTELEM)*16);
2570                                 }
2571                             }
2572
2573                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2574                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2575                             if(best_s.qscale != qp){
2576                                 if(s->mb_intra && s->dc_val[0]){
2577                                     for(i=0; i<6; i++){
2578                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2579                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(DCTELEM)*16);
2580                                     }
2581                                 }
2582                             }
2583                         }
2584                     }
2585                 }
2586                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2587                     int mx= s->b_direct_mv_table[xy][0];
2588                     int my= s->b_direct_mv_table[xy][1];
2589
2590                     backup_s.dquant = 0;
2591                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2592                     s->mb_intra= 0;
2593                     ff_mpeg4_set_direct_mv(s, mx, my);
2594                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2595                                  &dmin, &next_block, mx, my);
2596                 }
2597                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2598                     backup_s.dquant = 0;
2599                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2600                     s->mb_intra= 0;
2601                     ff_mpeg4_set_direct_mv(s, 0, 0);
2602                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2603                                  &dmin, &next_block, 0, 0);
2604                 }
2605                 if(!best_s.mb_intra && s->flags2&CODEC_FLAG2_SKIP_RD){
2606                     int coded=0;
2607                     for(i=0; i<6; i++)
2608                         coded |= s->block_last_index[i];
2609                     if(coded){
2610                         int mx,my;
2611                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2612                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
2613                             mx=my=0; //FIXME find the one we actually used
2614                             ff_mpeg4_set_direct_mv(s, mx, my);
2615                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2616                             mx= s->mv[1][0][0];
2617                             my= s->mv[1][0][1];
2618                         }else{
2619                             mx= s->mv[0][0][0];
2620                             my= s->mv[0][0][1];
2621                         }
2622
2623                         s->mv_dir= best_s.mv_dir;
2624                         s->mv_type = best_s.mv_type;
2625                         s->mb_intra= 0;
2626 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2627                         s->mv[0][0][1] = best_s.mv[0][0][1];
2628                         s->mv[1][0][0] = best_s.mv[1][0][0];
2629                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2630                         backup_s.dquant= 0;
2631                         s->skipdct=1;
2632                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2633                                         &dmin, &next_block, mx, my);
2634                         s->skipdct=0;
2635                     }
2636                 }
2637
2638                 s->current_picture.f.qscale_table[xy] = best_s.qscale;
2639
2640                 copy_context_after_encode(s, &best_s, -1);
2641
2642                 pb_bits_count= put_bits_count(&s->pb);
2643                 flush_put_bits(&s->pb);
2644                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2645                 s->pb= backup_s.pb;
2646
2647                 if(s->data_partitioning){
2648                     pb2_bits_count= put_bits_count(&s->pb2);
2649                     flush_put_bits(&s->pb2);
2650                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2651                     s->pb2= backup_s.pb2;
2652
2653                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2654                     flush_put_bits(&s->tex_pb);
2655                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2656                     s->tex_pb= backup_s.tex_pb;
2657                 }
2658                 s->last_bits= put_bits_count(&s->pb);
2659
2660                 if (CONFIG_H263_ENCODER &&
2661                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2662                     ff_h263_update_motion_val(s);
2663
2664                 if(next_block==0){ //FIXME 16 vs linesize16
2665                     s->dsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
2666                     s->dsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2667                     s->dsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2668                 }
2669
2670                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
2671                     MPV_decode_mb(s, s->block);
2672             } else {
2673                 int motion_x = 0, motion_y = 0;
2674                 s->mv_type=MV_TYPE_16X16;
2675                 // only one MB-Type possible
2676
2677                 switch(mb_type){
2678                 case CANDIDATE_MB_TYPE_INTRA:
2679                     s->mv_dir = 0;
2680                     s->mb_intra= 1;
2681                     motion_x= s->mv[0][0][0] = 0;
2682                     motion_y= s->mv[0][0][1] = 0;
2683                     break;
2684                 case CANDIDATE_MB_TYPE_INTER:
2685                     s->mv_dir = MV_DIR_FORWARD;
2686                     s->mb_intra= 0;
2687                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
2688                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
2689                     break;
2690                 case CANDIDATE_MB_TYPE_INTER_I:
2691                     s->mv_dir = MV_DIR_FORWARD;
2692                     s->mv_type = MV_TYPE_FIELD;
2693                     s->mb_intra= 0;
2694                     for(i=0; i<2; i++){
2695                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2696                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2697                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2698                     }
2699                     break;
2700                 case CANDIDATE_MB_TYPE_INTER4V:
2701                     s->mv_dir = MV_DIR_FORWARD;
2702                     s->mv_type = MV_TYPE_8X8;
2703                     s->mb_intra= 0;
2704                     for(i=0; i<4; i++){
2705                         s->mv[0][i][0] = s->current_picture.f.motion_val[0][s->block_index[i]][0];
2706                         s->mv[0][i][1] = s->current_picture.f.motion_val[0][s->block_index[i]][1];
2707                     }
2708                     break;
2709                 case CANDIDATE_MB_TYPE_DIRECT:
2710                     if (CONFIG_MPEG4_ENCODER) {
2711                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2712                         s->mb_intra= 0;
2713                         motion_x=s->b_direct_mv_table[xy][0];
2714                         motion_y=s->b_direct_mv_table[xy][1];
2715                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
2716                     }
2717                     break;
2718                 case CANDIDATE_MB_TYPE_DIRECT0:
2719                     if (CONFIG_MPEG4_ENCODER) {
2720                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2721                         s->mb_intra= 0;
2722                         ff_mpeg4_set_direct_mv(s, 0, 0);
2723                     }
2724                     break;
2725                 case CANDIDATE_MB_TYPE_BIDIR:
2726                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2727                     s->mb_intra= 0;
2728                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2729                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2730                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2731                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2732                     break;
2733                 case CANDIDATE_MB_TYPE_BACKWARD:
2734                     s->mv_dir = MV_DIR_BACKWARD;
2735                     s->mb_intra= 0;
2736                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2737                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2738                     break;
2739                 case CANDIDATE_MB_TYPE_FORWARD:
2740                     s->mv_dir = MV_DIR_FORWARD;
2741                     s->mb_intra= 0;
2742                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2743                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2744 //                    printf(" %d %d ", motion_x, motion_y);
2745                     break;
2746                 case CANDIDATE_MB_TYPE_FORWARD_I:
2747                     s->mv_dir = MV_DIR_FORWARD;
2748                     s->mv_type = MV_TYPE_FIELD;
2749                     s->mb_intra= 0;
2750                     for(i=0; i<2; i++){
2751                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2752                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2753                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2754                     }
2755                     break;
2756                 case CANDIDATE_MB_TYPE_BACKWARD_I:
2757                     s->mv_dir = MV_DIR_BACKWARD;
2758                     s->mv_type = MV_TYPE_FIELD;
2759                     s->mb_intra= 0;
2760                     for(i=0; i<2; i++){
2761                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2762                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2763                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2764                     }
2765                     break;
2766                 case CANDIDATE_MB_TYPE_BIDIR_I:
2767                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2768                     s->mv_type = MV_TYPE_FIELD;
2769                     s->mb_intra= 0;
2770                     for(dir=0; dir<2; dir++){
2771                         for(i=0; i<2; i++){
2772                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2773                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2774                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2775                         }
2776                     }
2777                     break;
2778                 default:
2779                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
2780                 }
2781
2782                 encode_mb(s, motion_x, motion_y);
2783
2784                 // RAL: Update last macroblock type
2785                 s->last_mv_dir = s->mv_dir;
2786
2787                 if (CONFIG_H263_ENCODER &&
2788                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2789                     ff_h263_update_motion_val(s);
2790
2791                 MPV_decode_mb(s, s->block);
2792             }
2793
2794             /* clean the MV table in IPS frames for direct mode in B frames */
2795             if(s->mb_intra /* && I,P,S_TYPE */){
2796                 s->p_mv_table[xy][0]=0;
2797                 s->p_mv_table[xy][1]=0;
2798             }
2799
2800             if(s->flags&CODEC_FLAG_PSNR){
2801                 int w= 16;
2802                 int h= 16;
2803
2804                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2805                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2806
2807                 s->current_picture.f.error[0] += sse(
2808                     s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
2809                     s->dest[0], w, h, s->linesize);
2810                 s->current_picture.f.error[1] += sse(
2811                     s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2812                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2813                 s->current_picture.f.error[2] += sse(
2814                     s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2815                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2816             }
2817             if(s->loop_filter){
2818                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
2819                     ff_h263_loop_filter(s);
2820             }
2821 //printf("MB %d %d bits\n", s->mb_x+s->mb_y*s->mb_stride, put_bits_count(&s->pb));
2822         }
2823     }
2824
2825     //not beautiful here but we must write it before flushing so it has to be here
2826     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
2827         msmpeg4_encode_ext_header(s);
2828
2829     write_slice_end(s);
2830
2831     /* Send the last GOB if RTP */
2832     if (s->avctx->rtp_callback) {
2833         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
2834         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
2835         /* Call the RTP callback to send the last GOB */
2836         emms_c();
2837         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
2838     }
2839
2840     return 0;
2841 }
2842
2843 #define MERGE(field) dst->field += src->field; src->field=0
2844 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
2845     MERGE(me.scene_change_score);
2846     MERGE(me.mc_mb_var_sum_temp);
2847     MERGE(me.mb_var_sum_temp);
2848 }
2849
2850 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
2851     int i;
2852
2853     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
2854     MERGE(dct_count[1]);
2855     MERGE(mv_bits);
2856     MERGE(i_tex_bits);
2857     MERGE(p_tex_bits);
2858     MERGE(i_count);
2859     MERGE(f_count);
2860     MERGE(b_count);
2861     MERGE(skip_count);
2862     MERGE(misc_bits);
2863     MERGE(error_count);
2864     MERGE(padding_bug_score);
2865     MERGE(current_picture.f.error[0]);
2866     MERGE(current_picture.f.error[1]);
2867     MERGE(current_picture.f.error[2]);
2868
2869     if(dst->avctx->noise_reduction){
2870         for(i=0; i<64; i++){
2871             MERGE(dct_error_sum[0][i]);
2872             MERGE(dct_error_sum[1][i]);
2873         }
2874     }
2875
2876     assert(put_bits_count(&src->pb) % 8 ==0);
2877     assert(put_bits_count(&dst->pb) % 8 ==0);
2878     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
2879     flush_put_bits(&dst->pb);
2880 }
2881
2882 static int estimate_qp(MpegEncContext *s, int dry_run){
2883     if (s->next_lambda){
2884         s->current_picture_ptr->f.quality =
2885         s->current_picture.f.quality = s->next_lambda;
2886         if(!dry_run) s->next_lambda= 0;
2887     } else if (!s->fixed_qscale) {
2888         s->current_picture_ptr->f.quality =
2889         s->current_picture.f.quality = ff_rate_estimate_qscale(s, dry_run);
2890         if (s->current_picture.f.quality < 0)
2891             return -1;
2892     }
2893
2894     if(s->adaptive_quant){
2895         switch(s->codec_id){
2896         case CODEC_ID_MPEG4:
2897             if (CONFIG_MPEG4_ENCODER)
2898                 ff_clean_mpeg4_qscales(s);
2899             break;
2900         case CODEC_ID_H263:
2901         case CODEC_ID_H263P:
2902         case CODEC_ID_FLV1:
2903             if (CONFIG_H263_ENCODER)
2904                 ff_clean_h263_qscales(s);
2905             break;
2906         default:
2907             ff_init_qscale_tab(s);
2908         }
2909
2910         s->lambda= s->lambda_table[0];
2911         //FIXME broken
2912     }else
2913         s->lambda = s->current_picture.f.quality;
2914 //printf("%d %d\n", s->avctx->global_quality, s->current_picture.quality);
2915     update_qscale(s);
2916     return 0;
2917 }
2918
2919 /* must be called before writing the header */
2920 static void set_frame_distances(MpegEncContext * s){
2921     assert(s->current_picture_ptr->f.pts != AV_NOPTS_VALUE);
2922     s->time = s->current_picture_ptr->f.pts * s->avctx->time_base.num;
2923
2924     if(s->pict_type==AV_PICTURE_TYPE_B){
2925         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
2926         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
2927     }else{
2928         s->pp_time= s->time - s->last_non_b_time;
2929         s->last_non_b_time= s->time;
2930         assert(s->picture_number==0 || s->pp_time > 0);
2931     }
2932 }
2933
2934 static int encode_picture(MpegEncContext *s, int picture_number)
2935 {
2936     int i;
2937     int bits;
2938     int context_count = s->avctx->thread_count;
2939
2940     s->picture_number = picture_number;
2941
2942     /* Reset the average MB variance */
2943     s->me.mb_var_sum_temp    =
2944     s->me.mc_mb_var_sum_temp = 0;
2945
2946     /* we need to initialize some time vars before we can encode b-frames */
2947     // RAL: Condition added for MPEG1VIDEO
2948     if (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
2949         set_frame_distances(s);
2950     if(CONFIG_MPEG4_ENCODER && s->codec_id == CODEC_ID_MPEG4)
2951         ff_set_mpeg4_time(s);
2952
2953     s->me.scene_change_score=0;
2954
2955 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
2956
2957     if(s->pict_type==AV_PICTURE_TYPE_I){
2958         if(s->msmpeg4_version >= 3) s->no_rounding=1;
2959         else                        s->no_rounding=0;
2960     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
2961         if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
2962             s->no_rounding ^= 1;
2963     }
2964
2965     if(s->flags & CODEC_FLAG_PASS2){
2966         if (estimate_qp(s,1) < 0)
2967             return -1;
2968         ff_get_2pass_fcode(s);
2969     }else if(!(s->flags & CODEC_FLAG_QSCALE)){
2970         if(s->pict_type==AV_PICTURE_TYPE_B)
2971             s->lambda= s->last_lambda_for[s->pict_type];
2972         else
2973             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
2974         update_qscale(s);
2975     }
2976
2977     if(s->codec_id != CODEC_ID_AMV){
2978         if(s->q_chroma_intra_matrix   != s->q_intra_matrix  ) av_freep(&s->q_chroma_intra_matrix);
2979         if(s->q_chroma_intra_matrix16 != s->q_intra_matrix16) av_freep(&s->q_chroma_intra_matrix16);
2980         s->q_chroma_intra_matrix   = s->q_intra_matrix;
2981         s->q_chroma_intra_matrix16 = s->q_intra_matrix16;
2982     }
2983
2984     s->mb_intra=0; //for the rate distortion & bit compare functions
2985     for(i=1; i<context_count; i++){
2986         ff_update_duplicate_context(s->thread_context[i], s);
2987     }
2988
2989     if(ff_init_me(s)<0)
2990         return -1;
2991
2992     /* Estimate motion for every MB */
2993     if(s->pict_type != AV_PICTURE_TYPE_I){
2994         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
2995         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
2996         if(s->pict_type != AV_PICTURE_TYPE_B && s->avctx->me_threshold==0){
2997             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
2998                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
2999             }
3000         }
3001
3002         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3003     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3004         /* I-Frame */
3005         for(i=0; i<s->mb_stride*s->mb_height; i++)
3006             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3007
3008         if(!s->fixed_qscale){
3009             /* finding spatial complexity for I-frame rate control */
3010             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3011         }
3012     }
3013     for(i=1; i<context_count; i++){
3014         merge_context_after_me(s, s->thread_context[i]);
3015     }
3016     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3017     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3018     emms_c();
3019
3020     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
3021         s->pict_type= AV_PICTURE_TYPE_I;
3022         for(i=0; i<s->mb_stride*s->mb_height; i++)
3023             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3024 //printf("Scene change detected, encoding as I Frame %d %d\n", s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3025     }
3026
3027     if(!s->umvplus){
3028         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3029             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3030
3031             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3032                 int a,b;
3033                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3034                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3035                 s->f_code= FFMAX3(s->f_code, a, b);
3036             }
3037
3038             ff_fix_long_p_mvs(s);
3039             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3040             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3041                 int j;
3042                 for(i=0; i<2; i++){
3043                     for(j=0; j<2; j++)
3044                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3045                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3046                 }
3047             }
3048         }
3049
3050         if(s->pict_type==AV_PICTURE_TYPE_B){
3051             int a, b;
3052
3053             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3054             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3055             s->f_code = FFMAX(a, b);
3056
3057             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3058             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3059             s->b_code = FFMAX(a, b);
3060
3061             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3062             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3063             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3064             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3065             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3066                 int dir, j;
3067                 for(dir=0; dir<2; dir++){
3068                     for(i=0; i<2; i++){
3069                         for(j=0; j<2; j++){
3070                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3071                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3072                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3073                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3074                         }
3075                     }
3076                 }
3077             }
3078         }
3079     }
3080
3081     if (estimate_qp(s, 0) < 0)
3082         return -1;
3083
3084     if(s->qscale < 3 && s->max_qcoeff<=128 && s->pict_type==AV_PICTURE_TYPE_I && !(s->flags & CODEC_FLAG_QSCALE))
3085         s->qscale= 3; //reduce clipping problems
3086
3087     if (s->out_format == FMT_MJPEG) {
3088         /* for mjpeg, we do include qscale in the matrix */
3089         for(i=1;i<64;i++){
3090             int j= s->dsp.idct_permutation[i];
3091
3092             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
3093         }
3094         s->y_dc_scale_table=
3095         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3096         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3097         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
3098                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3099         s->qscale= 8;
3100     }
3101     if(s->codec_id == CODEC_ID_AMV){
3102         static const uint8_t y[32]={13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13};
3103         static const uint8_t c[32]={14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14};
3104         for(i=1;i<64;i++){
3105             int j= s->dsp.idct_permutation[ff_zigzag_direct[i]];
3106
3107             s->intra_matrix[j] = sp5x_quant_table[5*2+0][i];
3108             s->chroma_intra_matrix[j] = sp5x_quant_table[5*2+1][i];
3109         }
3110         s->y_dc_scale_table= y;
3111         s->c_dc_scale_table= c;
3112         s->intra_matrix[0] = 13;
3113         s->chroma_intra_matrix[0] = 14;
3114         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
3115                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3116         ff_convert_matrix(&s->dsp, s->q_chroma_intra_matrix, s->q_chroma_intra_matrix16,
3117                        s->chroma_intra_matrix, s->intra_quant_bias, 8, 8, 1);
3118         s->qscale= 8;
3119     }
3120
3121     //FIXME var duplication
3122     s->current_picture_ptr->f.key_frame =
3123     s->current_picture.f.key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3124     s->current_picture_ptr->f.pict_type =
3125     s->current_picture.f.pict_type = s->pict_type;
3126
3127     if (s->current_picture.f.key_frame)
3128         s->picture_in_gop_number=0;
3129
3130     s->last_bits= put_bits_count(&s->pb);
3131     switch(s->out_format) {
3132     case FMT_MJPEG:
3133         if (CONFIG_MJPEG_ENCODER)
3134             ff_mjpeg_encode_picture_header(s);
3135         break;
3136     case FMT_H261:
3137         if (CONFIG_H261_ENCODER)
3138             ff_h261_encode_picture_header(s, picture_number);
3139         break;
3140     case FMT_H263:
3141         if (CONFIG_WMV2_ENCODER && s->codec_id == CODEC_ID_WMV2)
3142             ff_wmv2_encode_picture_header(s, picture_number);
3143         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3144             msmpeg4_encode_picture_header(s, picture_number);
3145         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
3146             mpeg4_encode_picture_header(s, picture_number);
3147         else if (CONFIG_RV10_ENCODER && s->codec_id == CODEC_ID_RV10)
3148             rv10_encode_picture_header(s, picture_number);
3149         else if (CONFIG_RV20_ENCODER && s->codec_id == CODEC_ID_RV20)
3150             rv20_encode_picture_header(s, picture_number);
3151         else if (CONFIG_FLV_ENCODER && s->codec_id == CODEC_ID_FLV1)
3152             ff_flv_encode_picture_header(s, picture_number);
3153         else if (CONFIG_H263_ENCODER)
3154             h263_encode_picture_header(s, picture_number);
3155         break;
3156     case FMT_MPEG1:
3157         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
3158             mpeg1_encode_picture_header(s, picture_number);
3159         break;
3160     case FMT_H264:
3161         break;
3162     default:
3163         assert(0);
3164     }
3165     bits= put_bits_count(&s->pb);
3166     s->header_bits= bits - s->last_bits;
3167
3168     for(i=1; i<context_count; i++){
3169         update_duplicate_context_after_me(s->thread_context[i], s);
3170     }
3171     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3172     for(i=1; i<context_count; i++){
3173         merge_context_after_encode(s, s->thread_context[i]);
3174     }
3175     emms_c();
3176     return 0;
3177 }
3178
3179 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block){
3180     const int intra= s->mb_intra;
3181     int i;
3182
3183     s->dct_count[intra]++;
3184
3185     for(i=0; i<64; i++){
3186         int level= block[i];
3187
3188         if(level){
3189             if(level>0){
3190                 s->dct_error_sum[intra][i] += level;
3191                 level -= s->dct_offset[intra][i];
3192                 if(level<0) level=0;
3193             }else{
3194                 s->dct_error_sum[intra][i] -= level;
3195                 level += s->dct_offset[intra][i];
3196                 if(level>0) level=0;
3197             }
3198             block[i]= level;
3199         }
3200     }
3201 }
3202
3203 static int dct_quantize_trellis_c(MpegEncContext *s,
3204                                   DCTELEM *block, int n,
3205                                   int qscale, int *overflow){
3206     const int *qmat;
3207     const uint8_t *scantable= s->intra_scantable.scantable;
3208     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3209     int max=0;
3210     unsigned int threshold1, threshold2;
3211     int bias=0;
3212     int run_tab[65];
3213     int level_tab[65];
3214     int score_tab[65];
3215     int survivor[65];
3216     int survivor_count;
3217     int last_run=0;
3218     int last_level=0;
3219     int last_score= 0;
3220     int last_i;
3221     int coeff[2][64];
3222     int coeff_count[64];
3223     int qmul, qadd, start_i, last_non_zero, i, dc;
3224     const int esc_length= s->ac_esc_length;
3225     uint8_t * length;
3226     uint8_t * last_length;
3227     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3228
3229     s->dsp.fdct (block);
3230
3231     if(s->dct_error_sum)
3232         s->denoise_dct(s, block);
3233     qmul= qscale*16;
3234     qadd= ((qscale-1)|1)*8;
3235
3236     if (s->mb_intra) {
3237         int q;
3238         if (!s->h263_aic) {
3239             if (n < 4)
3240                 q = s->y_dc_scale;
3241             else
3242                 q = s->c_dc_scale;
3243             q = q << 3;
3244         } else{
3245             /* For AIC we skip quant/dequant of INTRADC */
3246             q = 1 << 3;
3247             qadd=0;
3248         }
3249
3250         /* note: block[0] is assumed to be positive */
3251         block[0] = (block[0] + (q >> 1)) / q;
3252         start_i = 1;
3253         last_non_zero = 0;
3254         qmat = n < 4 ? s->q_intra_matrix[qscale] : s->q_chroma_intra_matrix[qscale];
3255         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3256             bias= 1<<(QMAT_SHIFT-1);
3257         length     = s->intra_ac_vlc_length;
3258         last_length= s->intra_ac_vlc_last_length;
3259     } else {
3260         start_i = 0;
3261         last_non_zero = -1;
3262         qmat = s->q_inter_matrix[qscale];
3263         length     = s->inter_ac_vlc_length;
3264         last_length= s->inter_ac_vlc_last_length;
3265     }
3266     last_i= start_i;
3267
3268     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3269     threshold2= (threshold1<<1);
3270
3271     for(i=63; i>=start_i; i--) {
3272         const int j = scantable[i];
3273         int level = block[j] * qmat[j];
3274
3275         if(((unsigned)(level+threshold1))>threshold2){
3276             last_non_zero = i;
3277             break;
3278         }
3279     }
3280
3281     for(i=start_i; i<=last_non_zero; i++) {
3282         const int j = scantable[i];
3283         int level = block[j] * qmat[j];
3284
3285 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3286 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3287         if(((unsigned)(level+threshold1))>threshold2){
3288             if(level>0){
3289                 level= (bias + level)>>QMAT_SHIFT;
3290                 coeff[0][i]= level;
3291                 coeff[1][i]= level-1;
3292 //                coeff[2][k]= level-2;
3293             }else{
3294                 level= (bias - level)>>QMAT_SHIFT;
3295                 coeff[0][i]= -level;
3296                 coeff[1][i]= -level+1;
3297 //                coeff[2][k]= -level+2;
3298             }
3299             coeff_count[i]= FFMIN(level, 2);
3300             assert(coeff_count[i]);
3301             max |=level;
3302         }else{
3303             coeff[0][i]= (level>>31)|1;
3304             coeff_count[i]= 1;
3305         }
3306     }
3307
3308     *overflow= s->max_qcoeff < max; //overflow might have happened
3309
3310     if(last_non_zero < start_i){
3311         memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3312         return last_non_zero;
3313     }
3314
3315     score_tab[start_i]= 0;
3316     survivor[0]= start_i;
3317     survivor_count= 1;
3318
3319     for(i=start_i; i<=last_non_zero; i++){
3320         int level_index, j, zero_distortion;
3321         int dct_coeff= FFABS(block[ scantable[i] ]);
3322         int best_score=256*256*256*120;
3323
3324         if (   s->dsp.fdct == fdct_ifast
3325 #ifndef FAAN_POSTSCALE
3326             || s->dsp.fdct == ff_faandct
3327 #endif
3328            )
3329             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3330         zero_distortion= dct_coeff*dct_coeff;
3331
3332         for(level_index=0; level_index < coeff_count[i]; level_index++){
3333             int distortion;
3334             int level= coeff[level_index][i];
3335             const int alevel= FFABS(level);
3336             int unquant_coeff;
3337
3338             assert(level);
3339
3340             if(s->out_format == FMT_H263){
3341                 unquant_coeff= alevel*qmul + qadd;
3342             }else{ //MPEG1
3343                 j= s->dsp.idct_permutation[ scantable[i] ]; //FIXME optimize
3344                 if(s->mb_intra){
3345                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3346                         unquant_coeff =   (unquant_coeff - 1) | 1;
3347                 }else{
3348                         unquant_coeff = (((  alevel  << 1) + 1) * qscale&n