dsputil: Move draw_edges() to mpegvideoencdsp
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of Libav.
9  *
10  * Libav is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * Libav is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with Libav; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include <stdint.h>
31
32 #include "libavutil/internal.h"
33 #include "libavutil/intmath.h"
34 #include "libavutil/mathematics.h"
35 #include "libavutil/pixdesc.h"
36 #include "libavutil/opt.h"
37 #include "libavutil/timer.h"
38 #include "avcodec.h"
39 #include "dct.h"
40 #include "dsputil.h"
41 #include "idctdsp.h"
42 #include "mpeg12.h"
43 #include "mpegvideo.h"
44 #include "h261.h"
45 #include "h263.h"
46 #include "mjpegenc_common.h"
47 #include "mathops.h"
48 #include "mpegutils.h"
49 #include "mjpegenc.h"
50 #include "msmpeg4.h"
51 #include "qpeldsp.h"
52 #include "faandct.h"
53 #include "thread.h"
54 #include "aandcttab.h"
55 #include "flv.h"
56 #include "mpeg4video.h"
57 #include "internal.h"
58 #include "bytestream.h"
59 #include <limits.h>
60
61 static int encode_picture(MpegEncContext *s, int picture_number);
62 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
63 static int sse_mb(MpegEncContext *s);
64 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
65 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
66
67 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
68 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
69
70 const AVOption ff_mpv_generic_options[] = {
71     FF_MPV_COMMON_OPTS
72     { NULL },
73 };
74
75 void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64],
76                        uint16_t (*qmat16)[2][64],
77                        const uint16_t *quant_matrix,
78                        int bias, int qmin, int qmax, int intra)
79 {
80     DSPContext *dsp = &s->dsp;
81     int qscale;
82     int shift = 0;
83
84     for (qscale = qmin; qscale <= qmax; qscale++) {
85         int i;
86         if (dsp->fdct == ff_jpeg_fdct_islow_8 ||
87             dsp->fdct == ff_jpeg_fdct_islow_10 ||
88             dsp->fdct == ff_faandct) {
89             for (i = 0; i < 64; i++) {
90                 const int j = s->idsp.idct_permutation[i];
91                 /* 16 <= qscale * quant_matrix[i] <= 7905
92                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
93                  *             19952 <=              x  <= 249205026
94                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
95                  *           3444240 >= (1 << 36) / (x) >= 275 */
96
97                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
98                                         (qscale * quant_matrix[j]));
99             }
100         } else if (dsp->fdct == ff_fdct_ifast) {
101             for (i = 0; i < 64; i++) {
102                 const int j = s->idsp.idct_permutation[i];
103                 /* 16 <= qscale * quant_matrix[i] <= 7905
104                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
105                  *             19952 <=              x  <= 249205026
106                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
107                  *           3444240 >= (1 << 36) / (x) >= 275 */
108
109                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) /
110                                         (ff_aanscales[i] * qscale *
111                                          quant_matrix[j]));
112             }
113         } else {
114             for (i = 0; i < 64; i++) {
115                 const int j = s->idsp.idct_permutation[i];
116                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
117                  * Assume x = qscale * quant_matrix[i]
118                  * So             16 <=              x  <= 7905
119                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
120                  * so          32768 >= (1 << 19) / (x) >= 67 */
121                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
122                                         (qscale * quant_matrix[j]));
123                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
124                 //                    (qscale * quant_matrix[i]);
125                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) /
126                                        (qscale * quant_matrix[j]);
127
128                 if (qmat16[qscale][0][i] == 0 ||
129                     qmat16[qscale][0][i] == 128 * 256)
130                     qmat16[qscale][0][i] = 128 * 256 - 1;
131                 qmat16[qscale][1][i] =
132                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
133                                 qmat16[qscale][0][i]);
134             }
135         }
136
137         for (i = intra; i < 64; i++) {
138             int64_t max = 8191;
139             if (dsp->fdct == ff_fdct_ifast) {
140                 max = (8191LL * ff_aanscales[i]) >> 14;
141             }
142             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
143                 shift++;
144             }
145         }
146     }
147     if (shift) {
148         av_log(NULL, AV_LOG_INFO,
149                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
150                QMAT_SHIFT - shift);
151     }
152 }
153
154 static inline void update_qscale(MpegEncContext *s)
155 {
156     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
157                 (FF_LAMBDA_SHIFT + 7);
158     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
159
160     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
161                  FF_LAMBDA_SHIFT;
162 }
163
164 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
165 {
166     int i;
167
168     if (matrix) {
169         put_bits(pb, 1, 1);
170         for (i = 0; i < 64; i++) {
171             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
172         }
173     } else
174         put_bits(pb, 1, 0);
175 }
176
177 /**
178  * init s->current_picture.qscale_table from s->lambda_table
179  */
180 void ff_init_qscale_tab(MpegEncContext *s)
181 {
182     int8_t * const qscale_table = s->current_picture.qscale_table;
183     int i;
184
185     for (i = 0; i < s->mb_num; i++) {
186         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
187         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
188         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
189                                                   s->avctx->qmax);
190     }
191 }
192
193 static void update_duplicate_context_after_me(MpegEncContext *dst,
194                                               MpegEncContext *src)
195 {
196 #define COPY(a) dst->a= src->a
197     COPY(pict_type);
198     COPY(current_picture);
199     COPY(f_code);
200     COPY(b_code);
201     COPY(qscale);
202     COPY(lambda);
203     COPY(lambda2);
204     COPY(picture_in_gop_number);
205     COPY(gop_picture_number);
206     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
207     COPY(progressive_frame);    // FIXME don't set in encode_header
208     COPY(partitioned_frame);    // FIXME don't set in encode_header
209 #undef COPY
210 }
211
212 /**
213  * Set the given MpegEncContext to defaults for encoding.
214  * the changed fields will not depend upon the prior state of the MpegEncContext.
215  */
216 static void MPV_encode_defaults(MpegEncContext *s)
217 {
218     int i;
219     ff_MPV_common_defaults(s);
220
221     for (i = -16; i < 16; i++) {
222         default_fcode_tab[i + MAX_MV] = 1;
223     }
224     s->me.mv_penalty = default_mv_penalty;
225     s->fcode_tab     = default_fcode_tab;
226
227     s->input_picture_number  = 0;
228     s->picture_in_gop_number = 0;
229 }
230
231 /* init video encoder */
232 av_cold int ff_MPV_encode_init(AVCodecContext *avctx)
233 {
234     MpegEncContext *s = avctx->priv_data;
235     int i, ret, format_supported;
236
237     MPV_encode_defaults(s);
238
239     switch (avctx->codec_id) {
240     case AV_CODEC_ID_MPEG2VIDEO:
241         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
242             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
243             av_log(avctx, AV_LOG_ERROR,
244                    "only YUV420 and YUV422 are supported\n");
245             return -1;
246         }
247         break;
248     case AV_CODEC_ID_MJPEG:
249         format_supported = 0;
250         /* JPEG color space */
251         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
252             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
253             (avctx->color_range == AVCOL_RANGE_JPEG &&
254              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
255               avctx->pix_fmt == AV_PIX_FMT_YUV422P)))
256             format_supported = 1;
257         /* MPEG color space */
258         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
259                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
260                   avctx->pix_fmt == AV_PIX_FMT_YUV422P))
261             format_supported = 1;
262
263         if (!format_supported) {
264             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
265             return -1;
266         }
267         break;
268     default:
269         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
270             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
271             return -1;
272         }
273     }
274
275     switch (avctx->pix_fmt) {
276     case AV_PIX_FMT_YUVJ422P:
277     case AV_PIX_FMT_YUV422P:
278         s->chroma_format = CHROMA_422;
279         break;
280     case AV_PIX_FMT_YUVJ420P:
281     case AV_PIX_FMT_YUV420P:
282     default:
283         s->chroma_format = CHROMA_420;
284         break;
285     }
286
287     s->bit_rate = avctx->bit_rate;
288     s->width    = avctx->width;
289     s->height   = avctx->height;
290     if (avctx->gop_size > 600 &&
291         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
292         av_log(avctx, AV_LOG_ERROR,
293                "Warning keyframe interval too large! reducing it ...\n");
294         avctx->gop_size = 600;
295     }
296     s->gop_size     = avctx->gop_size;
297     s->avctx        = avctx;
298     s->flags        = avctx->flags;
299     s->flags2       = avctx->flags2;
300     if (avctx->max_b_frames > MAX_B_FRAMES) {
301         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
302                "is %d.\n", MAX_B_FRAMES);
303     }
304     s->max_b_frames = avctx->max_b_frames;
305     s->codec_id     = avctx->codec->id;
306     s->strict_std_compliance = avctx->strict_std_compliance;
307     s->quarter_sample     = (avctx->flags & CODEC_FLAG_QPEL) != 0;
308     s->mpeg_quant         = avctx->mpeg_quant;
309     s->rtp_mode           = !!avctx->rtp_payload_size;
310     s->intra_dc_precision = avctx->intra_dc_precision;
311     s->user_specified_pts = AV_NOPTS_VALUE;
312
313     if (s->gop_size <= 1) {
314         s->intra_only = 1;
315         s->gop_size   = 12;
316     } else {
317         s->intra_only = 0;
318     }
319
320     s->me_method = avctx->me_method;
321
322     /* Fixed QSCALE */
323     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
324
325     s->adaptive_quant = (s->avctx->lumi_masking ||
326                          s->avctx->dark_masking ||
327                          s->avctx->temporal_cplx_masking ||
328                          s->avctx->spatial_cplx_masking  ||
329                          s->avctx->p_masking      ||
330                          s->avctx->border_masking ||
331                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
332                         !s->fixed_qscale;
333
334     s->loop_filter      = !!(s->flags & CODEC_FLAG_LOOP_FILTER);
335
336     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
337         av_log(avctx, AV_LOG_ERROR,
338                "a vbv buffer size is needed, "
339                "for encoding with a maximum bitrate\n");
340         return -1;
341     }
342
343     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
344         av_log(avctx, AV_LOG_INFO,
345                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
346     }
347
348     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
349         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
350         return -1;
351     }
352
353     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
354         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
355         return -1;
356     }
357
358     if (avctx->rc_max_rate &&
359         avctx->rc_max_rate == avctx->bit_rate &&
360         avctx->rc_max_rate != avctx->rc_min_rate) {
361         av_log(avctx, AV_LOG_INFO,
362                "impossible bitrate constraints, this will fail\n");
363     }
364
365     if (avctx->rc_buffer_size &&
366         avctx->bit_rate * (int64_t)avctx->time_base.num >
367             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
368         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
369         return -1;
370     }
371
372     if (!s->fixed_qscale &&
373         avctx->bit_rate * av_q2d(avctx->time_base) >
374             avctx->bit_rate_tolerance) {
375         av_log(avctx, AV_LOG_ERROR,
376                "bitrate tolerance too small for bitrate\n");
377         return -1;
378     }
379
380     if (s->avctx->rc_max_rate &&
381         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
382         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
383          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
384         90000LL * (avctx->rc_buffer_size - 1) >
385             s->avctx->rc_max_rate * 0xFFFFLL) {
386         av_log(avctx, AV_LOG_INFO,
387                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
388                "specified vbv buffer is too large for the given bitrate!\n");
389     }
390
391     if ((s->flags & CODEC_FLAG_4MV)  && s->codec_id != AV_CODEC_ID_MPEG4 &&
392         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
393         s->codec_id != AV_CODEC_ID_FLV1) {
394         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
395         return -1;
396     }
397
398     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
399         av_log(avctx, AV_LOG_ERROR,
400                "OBMC is only supported with simple mb decision\n");
401         return -1;
402     }
403
404     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
405         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
406         return -1;
407     }
408
409     if (s->max_b_frames                    &&
410         s->codec_id != AV_CODEC_ID_MPEG4      &&
411         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
412         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
413         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
414         return -1;
415     }
416
417     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
418          s->codec_id == AV_CODEC_ID_H263  ||
419          s->codec_id == AV_CODEC_ID_H263P) &&
420         (avctx->sample_aspect_ratio.num > 255 ||
421          avctx->sample_aspect_ratio.den > 255)) {
422         av_log(avctx, AV_LOG_ERROR,
423                "Invalid pixel aspect ratio %i/%i, limit is 255/255\n",
424                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
425         return -1;
426     }
427
428     if ((s->flags & (CODEC_FLAG_INTERLACED_DCT | CODEC_FLAG_INTERLACED_ME)) &&
429         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
430         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
431         return -1;
432     }
433
434     // FIXME mpeg2 uses that too
435     if (s->mpeg_quant && s->codec_id != AV_CODEC_ID_MPEG4) {
436         av_log(avctx, AV_LOG_ERROR,
437                "mpeg2 style quantization not supported by codec\n");
438         return -1;
439     }
440
441     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
442         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
443         return -1;
444     }
445
446     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
447         s->avctx->mb_decision != FF_MB_DECISION_RD) {
448         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
449         return -1;
450     }
451
452     if (s->avctx->scenechange_threshold < 1000000000 &&
453         (s->flags & CODEC_FLAG_CLOSED_GOP)) {
454         av_log(avctx, AV_LOG_ERROR,
455                "closed gop with scene change detection are not supported yet, "
456                "set threshold to 1000000000\n");
457         return -1;
458     }
459
460     if (s->flags & CODEC_FLAG_LOW_DELAY) {
461         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
462             av_log(avctx, AV_LOG_ERROR,
463                   "low delay forcing is only available for mpeg2\n");
464             return -1;
465         }
466         if (s->max_b_frames != 0) {
467             av_log(avctx, AV_LOG_ERROR,
468                    "b frames cannot be used with low delay\n");
469             return -1;
470         }
471     }
472
473     if (s->q_scale_type == 1) {
474         if (avctx->qmax > 12) {
475             av_log(avctx, AV_LOG_ERROR,
476                    "non linear quant only supports qmax <= 12 currently\n");
477             return -1;
478         }
479     }
480
481     if (s->avctx->thread_count > 1         &&
482         s->codec_id != AV_CODEC_ID_MPEG4      &&
483         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
484         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
485         (s->codec_id != AV_CODEC_ID_H263P)) {
486         av_log(avctx, AV_LOG_ERROR,
487                "multi threaded encoding not supported by codec\n");
488         return -1;
489     }
490
491     if (s->avctx->thread_count < 1) {
492         av_log(avctx, AV_LOG_ERROR,
493                "automatic thread number detection not supported by codec,"
494                "patch welcome\n");
495         return -1;
496     }
497
498     if (s->avctx->thread_count > 1)
499         s->rtp_mode = 1;
500
501     if (!avctx->time_base.den || !avctx->time_base.num) {
502         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
503         return -1;
504     }
505
506     i = (INT_MAX / 2 + 128) >> 8;
507     if (avctx->mb_threshold >= i) {
508         av_log(avctx, AV_LOG_ERROR, "mb_threshold too large, max is %d\n",
509                i - 1);
510         return -1;
511     }
512
513     if (avctx->b_frame_strategy && (avctx->flags & CODEC_FLAG_PASS2)) {
514         av_log(avctx, AV_LOG_INFO,
515                "notice: b_frame_strategy only affects the first pass\n");
516         avctx->b_frame_strategy = 0;
517     }
518
519     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
520     if (i > 1) {
521         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
522         avctx->time_base.den /= i;
523         avctx->time_base.num /= i;
524         //return -1;
525     }
526
527     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
528         s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG) {
529         // (a + x * 3 / 8) / x
530         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
531         s->inter_quant_bias = 0;
532     } else {
533         s->intra_quant_bias = 0;
534         // (a - x / 4) / x
535         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
536     }
537
538     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
539         s->intra_quant_bias = avctx->intra_quant_bias;
540     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
541         s->inter_quant_bias = avctx->inter_quant_bias;
542
543     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
544         s->avctx->time_base.den > (1 << 16) - 1) {
545         av_log(avctx, AV_LOG_ERROR,
546                "timebase %d/%d not supported by MPEG 4 standard, "
547                "the maximum admitted value for the timebase denominator "
548                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
549                (1 << 16) - 1);
550         return -1;
551     }
552     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
553
554     switch (avctx->codec->id) {
555     case AV_CODEC_ID_MPEG1VIDEO:
556         s->out_format = FMT_MPEG1;
557         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
558         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
559         break;
560     case AV_CODEC_ID_MPEG2VIDEO:
561         s->out_format = FMT_MPEG1;
562         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
563         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
564         s->rtp_mode   = 1;
565         break;
566     case AV_CODEC_ID_MJPEG:
567         s->out_format = FMT_MJPEG;
568         s->intra_only = 1; /* force intra only for jpeg */
569         if (!CONFIG_MJPEG_ENCODER ||
570             ff_mjpeg_encode_init(s) < 0)
571             return -1;
572         avctx->delay = 0;
573         s->low_delay = 1;
574         break;
575     case AV_CODEC_ID_H261:
576         if (!CONFIG_H261_ENCODER)
577             return -1;
578         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
579             av_log(avctx, AV_LOG_ERROR,
580                    "The specified picture size of %dx%d is not valid for the "
581                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
582                     s->width, s->height);
583             return -1;
584         }
585         s->out_format = FMT_H261;
586         avctx->delay  = 0;
587         s->low_delay  = 1;
588         break;
589     case AV_CODEC_ID_H263:
590         if (!CONFIG_H263_ENCODER)
591         return -1;
592         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
593                              s->width, s->height) == 8) {
594             av_log(avctx, AV_LOG_INFO,
595                    "The specified picture size of %dx%d is not valid for "
596                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
597                    "352x288, 704x576, and 1408x1152."
598                    "Try H.263+.\n", s->width, s->height);
599             return -1;
600         }
601         s->out_format = FMT_H263;
602         avctx->delay  = 0;
603         s->low_delay  = 1;
604         break;
605     case AV_CODEC_ID_H263P:
606         s->out_format = FMT_H263;
607         s->h263_plus  = 1;
608         /* Fx */
609         s->h263_aic        = (avctx->flags & CODEC_FLAG_AC_PRED) ? 1 : 0;
610         s->modified_quant  = s->h263_aic;
611         s->loop_filter     = (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
612         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
613
614         /* /Fx */
615         /* These are just to be sure */
616         avctx->delay = 0;
617         s->low_delay = 1;
618         break;
619     case AV_CODEC_ID_FLV1:
620         s->out_format      = FMT_H263;
621         s->h263_flv        = 2; /* format = 1; 11-bit codes */
622         s->unrestricted_mv = 1;
623         s->rtp_mode  = 0; /* don't allow GOB */
624         avctx->delay = 0;
625         s->low_delay = 1;
626         break;
627     case AV_CODEC_ID_RV10:
628         s->out_format = FMT_H263;
629         avctx->delay  = 0;
630         s->low_delay  = 1;
631         break;
632     case AV_CODEC_ID_RV20:
633         s->out_format      = FMT_H263;
634         avctx->delay       = 0;
635         s->low_delay       = 1;
636         s->modified_quant  = 1;
637         s->h263_aic        = 1;
638         s->h263_plus       = 1;
639         s->loop_filter     = 1;
640         s->unrestricted_mv = 0;
641         break;
642     case AV_CODEC_ID_MPEG4:
643         s->out_format      = FMT_H263;
644         s->h263_pred       = 1;
645         s->unrestricted_mv = 1;
646         s->low_delay       = s->max_b_frames ? 0 : 1;
647         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
648         break;
649     case AV_CODEC_ID_MSMPEG4V2:
650         s->out_format      = FMT_H263;
651         s->h263_pred       = 1;
652         s->unrestricted_mv = 1;
653         s->msmpeg4_version = 2;
654         avctx->delay       = 0;
655         s->low_delay       = 1;
656         break;
657     case AV_CODEC_ID_MSMPEG4V3:
658         s->out_format        = FMT_H263;
659         s->h263_pred         = 1;
660         s->unrestricted_mv   = 1;
661         s->msmpeg4_version   = 3;
662         s->flipflop_rounding = 1;
663         avctx->delay         = 0;
664         s->low_delay         = 1;
665         break;
666     case AV_CODEC_ID_WMV1:
667         s->out_format        = FMT_H263;
668         s->h263_pred         = 1;
669         s->unrestricted_mv   = 1;
670         s->msmpeg4_version   = 4;
671         s->flipflop_rounding = 1;
672         avctx->delay         = 0;
673         s->low_delay         = 1;
674         break;
675     case AV_CODEC_ID_WMV2:
676         s->out_format        = FMT_H263;
677         s->h263_pred         = 1;
678         s->unrestricted_mv   = 1;
679         s->msmpeg4_version   = 5;
680         s->flipflop_rounding = 1;
681         avctx->delay         = 0;
682         s->low_delay         = 1;
683         break;
684     default:
685         return -1;
686     }
687
688     avctx->has_b_frames = !s->low_delay;
689
690     s->encoding = 1;
691
692     s->progressive_frame    =
693     s->progressive_sequence = !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT |
694                                                 CODEC_FLAG_INTERLACED_ME) ||
695                                 s->alternate_scan);
696
697     /* init */
698     if (ff_MPV_common_init(s) < 0)
699         return -1;
700
701     if (ARCH_X86)
702         ff_MPV_encode_init_x86(s);
703
704     ff_mpegvideoencdsp_init(&s->mpvencdsp, avctx);
705     ff_qpeldsp_init(&s->qdsp);
706
707     s->avctx->coded_frame = s->current_picture.f;
708
709     if (s->msmpeg4_version) {
710         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
711                           2 * 2 * (MAX_LEVEL + 1) *
712                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
713     }
714     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
715
716     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
717     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
718     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
719     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
720     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
721                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
722     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
723                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
724
725     if (s->avctx->noise_reduction) {
726         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
727                           2 * 64 * sizeof(uint16_t), fail);
728     }
729
730     if (CONFIG_H263_ENCODER)
731         ff_h263dsp_init(&s->h263dsp);
732     if (!s->dct_quantize)
733         s->dct_quantize = ff_dct_quantize_c;
734     if (!s->denoise_dct)
735         s->denoise_dct  = denoise_dct_c;
736     s->fast_dct_quantize = s->dct_quantize;
737     if (avctx->trellis)
738         s->dct_quantize  = dct_quantize_trellis_c;
739
740     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
741         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
742
743     s->quant_precision = 5;
744
745     ff_set_cmp(&s->dsp, s->dsp.ildct_cmp, s->avctx->ildct_cmp);
746     ff_set_cmp(&s->dsp, s->dsp.frame_skip_cmp, s->avctx->frame_skip_cmp);
747
748     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
749         ff_h261_encode_init(s);
750     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
751         ff_h263_encode_init(s);
752     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
753         ff_msmpeg4_encode_init(s);
754     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
755         && s->out_format == FMT_MPEG1)
756         ff_mpeg1_encode_init(s);
757
758     /* init q matrix */
759     for (i = 0; i < 64; i++) {
760         int j = s->idsp.idct_permutation[i];
761         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
762             s->mpeg_quant) {
763             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
764             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
765         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
766             s->intra_matrix[j] =
767             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
768         } else {
769             /* mpeg1/2 */
770             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
771             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
772         }
773         if (s->avctx->intra_matrix)
774             s->intra_matrix[j] = s->avctx->intra_matrix[i];
775         if (s->avctx->inter_matrix)
776             s->inter_matrix[j] = s->avctx->inter_matrix[i];
777     }
778
779     /* precompute matrix */
780     /* for mjpeg, we do include qscale in the matrix */
781     if (s->out_format != FMT_MJPEG) {
782         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
783                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
784                           31, 1);
785         ff_convert_matrix(s, s->q_inter_matrix, s->q_inter_matrix16,
786                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
787                           31, 0);
788     }
789
790     if (ff_rate_control_init(s) < 0)
791         return -1;
792
793 #if FF_API_ERROR_RATE
794     FF_DISABLE_DEPRECATION_WARNINGS
795     if (avctx->error_rate)
796         s->error_rate = avctx->error_rate;
797     FF_ENABLE_DEPRECATION_WARNINGS;
798 #endif
799
800 #if FF_API_NORMALIZE_AQP
801     FF_DISABLE_DEPRECATION_WARNINGS
802     if (avctx->flags & CODEC_FLAG_NORMALIZE_AQP)
803         s->mpv_flags |= FF_MPV_FLAG_NAQ;
804     FF_ENABLE_DEPRECATION_WARNINGS;
805 #endif
806
807 #if FF_API_MV0
808     FF_DISABLE_DEPRECATION_WARNINGS
809     if (avctx->flags & CODEC_FLAG_MV0)
810         s->mpv_flags |= FF_MPV_FLAG_MV0;
811     FF_ENABLE_DEPRECATION_WARNINGS
812 #endif
813
814     if (avctx->b_frame_strategy == 2) {
815         for (i = 0; i < s->max_b_frames + 2; i++) {
816             s->tmp_frames[i] = av_frame_alloc();
817             if (!s->tmp_frames[i])
818                 return AVERROR(ENOMEM);
819
820             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
821             s->tmp_frames[i]->width  = s->width  >> avctx->brd_scale;
822             s->tmp_frames[i]->height = s->height >> avctx->brd_scale;
823
824             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
825             if (ret < 0)
826                 return ret;
827         }
828     }
829
830     return 0;
831 fail:
832     ff_MPV_encode_end(avctx);
833     return AVERROR_UNKNOWN;
834 }
835
836 av_cold int ff_MPV_encode_end(AVCodecContext *avctx)
837 {
838     MpegEncContext *s = avctx->priv_data;
839     int i;
840
841     ff_rate_control_uninit(s);
842
843     ff_MPV_common_end(s);
844     if (CONFIG_MJPEG_ENCODER &&
845         s->out_format == FMT_MJPEG)
846         ff_mjpeg_encode_close(s);
847
848     av_freep(&avctx->extradata);
849
850     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
851         av_frame_free(&s->tmp_frames[i]);
852
853     ff_free_picture_tables(&s->new_picture);
854     ff_mpeg_unref_picture(s, &s->new_picture);
855
856     av_freep(&s->avctx->stats_out);
857     av_freep(&s->ac_stats);
858
859     av_freep(&s->q_intra_matrix);
860     av_freep(&s->q_inter_matrix);
861     av_freep(&s->q_intra_matrix16);
862     av_freep(&s->q_inter_matrix16);
863     av_freep(&s->input_picture);
864     av_freep(&s->reordered_input_picture);
865     av_freep(&s->dct_offset);
866
867     return 0;
868 }
869
870 static int get_sae(uint8_t *src, int ref, int stride)
871 {
872     int x,y;
873     int acc = 0;
874
875     for (y = 0; y < 16; y++) {
876         for (x = 0; x < 16; x++) {
877             acc += FFABS(src[x + y * stride] - ref);
878         }
879     }
880
881     return acc;
882 }
883
884 static int get_intra_count(MpegEncContext *s, uint8_t *src,
885                            uint8_t *ref, int stride)
886 {
887     int x, y, w, h;
888     int acc = 0;
889
890     w = s->width  & ~15;
891     h = s->height & ~15;
892
893     for (y = 0; y < h; y += 16) {
894         for (x = 0; x < w; x += 16) {
895             int offset = x + y * stride;
896             int sad  = s->dsp.sad[0](NULL, src + offset, ref + offset, stride,
897                                      16);
898             int mean = (s->mpvencdsp.pix_sum(src + offset, stride) + 128) >> 8;
899             int sae  = get_sae(src + offset, mean, stride);
900
901             acc += sae + 500 < sad;
902         }
903     }
904     return acc;
905 }
906
907
908 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
909 {
910     Picture *pic = NULL;
911     int64_t pts;
912     int i, display_picture_number = 0, ret;
913     const int encoding_delay = s->max_b_frames ? s->max_b_frames :
914                                                  (s->low_delay ? 0 : 1);
915     int direct = 1;
916
917     if (pic_arg) {
918         pts = pic_arg->pts;
919         display_picture_number = s->input_picture_number++;
920
921         if (pts != AV_NOPTS_VALUE) {
922             if (s->user_specified_pts != AV_NOPTS_VALUE) {
923                 int64_t time = pts;
924                 int64_t last = s->user_specified_pts;
925
926                 if (time <= last) {
927                     av_log(s->avctx, AV_LOG_ERROR,
928                            "Error, Invalid timestamp=%"PRId64", "
929                            "last=%"PRId64"\n", pts, s->user_specified_pts);
930                     return -1;
931                 }
932
933                 if (!s->low_delay && display_picture_number == 1)
934                     s->dts_delta = time - last;
935             }
936             s->user_specified_pts = pts;
937         } else {
938             if (s->user_specified_pts != AV_NOPTS_VALUE) {
939                 s->user_specified_pts =
940                 pts = s->user_specified_pts + 1;
941                 av_log(s->avctx, AV_LOG_INFO,
942                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
943                        pts);
944             } else {
945                 pts = display_picture_number;
946             }
947         }
948     }
949
950     if (pic_arg) {
951         if (!pic_arg->buf[0]);
952             direct = 0;
953         if (pic_arg->linesize[0] != s->linesize)
954             direct = 0;
955         if (pic_arg->linesize[1] != s->uvlinesize)
956             direct = 0;
957         if (pic_arg->linesize[2] != s->uvlinesize)
958             direct = 0;
959
960         av_dlog(s->avctx, "%d %d %td %td\n", pic_arg->linesize[0],
961                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
962
963         if (direct) {
964             i = ff_find_unused_picture(s, 1);
965             if (i < 0)
966                 return i;
967
968             pic = &s->picture[i];
969             pic->reference = 3;
970
971             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
972                 return ret;
973             if (ff_alloc_picture(s, pic, 1) < 0) {
974                 return -1;
975             }
976         } else {
977             i = ff_find_unused_picture(s, 0);
978             if (i < 0)
979                 return i;
980
981             pic = &s->picture[i];
982             pic->reference = 3;
983
984             if (ff_alloc_picture(s, pic, 0) < 0) {
985                 return -1;
986             }
987
988             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
989                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
990                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
991                 // empty
992             } else {
993                 int h_chroma_shift, v_chroma_shift;
994                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
995                                                  &h_chroma_shift,
996                                                  &v_chroma_shift);
997
998                 for (i = 0; i < 3; i++) {
999                     int src_stride = pic_arg->linesize[i];
1000                     int dst_stride = i ? s->uvlinesize : s->linesize;
1001                     int h_shift = i ? h_chroma_shift : 0;
1002                     int v_shift = i ? v_chroma_shift : 0;
1003                     int w = s->width  >> h_shift;
1004                     int h = s->height >> v_shift;
1005                     uint8_t *src = pic_arg->data[i];
1006                     uint8_t *dst = pic->f->data[i];
1007
1008                     if (!s->avctx->rc_buffer_size)
1009                         dst += INPLACE_OFFSET;
1010
1011                     if (src_stride == dst_stride)
1012                         memcpy(dst, src, src_stride * h);
1013                     else {
1014                         while (h--) {
1015                             memcpy(dst, src, w);
1016                             dst += dst_stride;
1017                             src += src_stride;
1018                         }
1019                     }
1020                 }
1021             }
1022         }
1023         ret = av_frame_copy_props(pic->f, pic_arg);
1024         if (ret < 0)
1025             return ret;
1026
1027         pic->f->display_picture_number = display_picture_number;
1028         pic->f->pts = pts; // we set this here to avoid modifiying pic_arg
1029     }
1030
1031     /* shift buffer entries */
1032     for (i = 1; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1033         s->input_picture[i - 1] = s->input_picture[i];
1034
1035     s->input_picture[encoding_delay] = (Picture*) pic;
1036
1037     return 0;
1038 }
1039
1040 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1041 {
1042     int x, y, plane;
1043     int score = 0;
1044     int64_t score64 = 0;
1045
1046     for (plane = 0; plane < 3; plane++) {
1047         const int stride = p->f->linesize[plane];
1048         const int bw = plane ? 1 : 2;
1049         for (y = 0; y < s->mb_height * bw; y++) {
1050             for (x = 0; x < s->mb_width * bw; x++) {
1051                 int off = p->shared ? 0 : 16;
1052                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1053                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1054                 int v   = s->dsp.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1055
1056                 switch (s->avctx->frame_skip_exp) {
1057                 case 0: score    =  FFMAX(score, v);          break;
1058                 case 1: score   += FFABS(v);                  break;
1059                 case 2: score   += v * v;                     break;
1060                 case 3: score64 += FFABS(v * v * (int64_t)v); break;
1061                 case 4: score64 += v * v * (int64_t)(v * v);  break;
1062                 }
1063             }
1064         }
1065     }
1066
1067     if (score)
1068         score64 = score;
1069
1070     if (score64 < s->avctx->frame_skip_threshold)
1071         return 1;
1072     if (score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda) >> 8))
1073         return 1;
1074     return 0;
1075 }
1076
1077 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1078 {
1079     AVPacket pkt = { 0 };
1080     int ret, got_output;
1081
1082     av_init_packet(&pkt);
1083     ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
1084     if (ret < 0)
1085         return ret;
1086
1087     ret = pkt.size;
1088     av_free_packet(&pkt);
1089     return ret;
1090 }
1091
1092 static int estimate_best_b_count(MpegEncContext *s)
1093 {
1094     AVCodec *codec    = avcodec_find_encoder(s->avctx->codec_id);
1095     AVCodecContext *c = avcodec_alloc_context3(NULL);
1096     const int scale = s->avctx->brd_scale;
1097     int i, j, out_size, p_lambda, b_lambda, lambda2;
1098     int64_t best_rd  = INT64_MAX;
1099     int best_b_count = -1;
1100
1101     assert(scale >= 0 && scale <= 3);
1102
1103     //emms_c();
1104     //s->next_picture_ptr->quality;
1105     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1106     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1107     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1108     if (!b_lambda) // FIXME we should do this somewhere else
1109         b_lambda = p_lambda;
1110     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1111                FF_LAMBDA_SHIFT;
1112
1113     c->width        = s->width  >> scale;
1114     c->height       = s->height >> scale;
1115     c->flags        = CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR;
1116     c->flags       |= s->avctx->flags & CODEC_FLAG_QPEL;
1117     c->mb_decision  = s->avctx->mb_decision;
1118     c->me_cmp       = s->avctx->me_cmp;
1119     c->mb_cmp       = s->avctx->mb_cmp;
1120     c->me_sub_cmp   = s->avctx->me_sub_cmp;
1121     c->pix_fmt      = AV_PIX_FMT_YUV420P;
1122     c->time_base    = s->avctx->time_base;
1123     c->max_b_frames = s->max_b_frames;
1124
1125     if (avcodec_open2(c, codec, NULL) < 0)
1126         return -1;
1127
1128     for (i = 0; i < s->max_b_frames + 2; i++) {
1129         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1130                                                 s->next_picture_ptr;
1131
1132         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1133             pre_input = *pre_input_ptr;
1134
1135             if (!pre_input.shared && i) {
1136                 pre_input.f->data[0] += INPLACE_OFFSET;
1137                 pre_input.f->data[1] += INPLACE_OFFSET;
1138                 pre_input.f->data[2] += INPLACE_OFFSET;
1139             }
1140
1141             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[0],
1142                                        s->tmp_frames[i]->linesize[0],
1143                                        pre_input.f->data[0],
1144                                        pre_input.f->linesize[0],
1145                                        c->width, c->height);
1146             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[1],
1147                                        s->tmp_frames[i]->linesize[1],
1148                                        pre_input.f->data[1],
1149                                        pre_input.f->linesize[1],
1150                                        c->width >> 1, c->height >> 1);
1151             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[2],
1152                                        s->tmp_frames[i]->linesize[2],
1153                                        pre_input.f->data[2],
1154                                        pre_input.f->linesize[2],
1155                                        c->width >> 1, c->height >> 1);
1156         }
1157     }
1158
1159     for (j = 0; j < s->max_b_frames + 1; j++) {
1160         int64_t rd = 0;
1161
1162         if (!s->input_picture[j])
1163             break;
1164
1165         c->error[0] = c->error[1] = c->error[2] = 0;
1166
1167         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1168         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1169
1170         out_size = encode_frame(c, s->tmp_frames[0]);
1171
1172         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1173
1174         for (i = 0; i < s->max_b_frames + 1; i++) {
1175             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1176
1177             s->tmp_frames[i + 1]->pict_type = is_p ?
1178                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1179             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1180
1181             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1182
1183             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1184         }
1185
1186         /* get the delayed frames */
1187         while (out_size) {
1188             out_size = encode_frame(c, NULL);
1189             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1190         }
1191
1192         rd += c->error[0] + c->error[1] + c->error[2];
1193
1194         if (rd < best_rd) {
1195             best_rd = rd;
1196             best_b_count = j;
1197         }
1198     }
1199
1200     avcodec_close(c);
1201     av_freep(&c);
1202
1203     return best_b_count;
1204 }
1205
1206 static int select_input_picture(MpegEncContext *s)
1207 {
1208     int i, ret;
1209
1210     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1211         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1212     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1213
1214     /* set next picture type & ordering */
1215     if (s->reordered_input_picture[0] == NULL && s->input_picture[0]) {
1216         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1217             s->next_picture_ptr == NULL || s->intra_only) {
1218             s->reordered_input_picture[0] = s->input_picture[0];
1219             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1220             s->reordered_input_picture[0]->f->coded_picture_number =
1221                 s->coded_picture_number++;
1222         } else {
1223             int b_frames;
1224
1225             if (s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor) {
1226                 if (s->picture_in_gop_number < s->gop_size &&
1227                     skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1228                     // FIXME check that te gop check above is +-1 correct
1229                     av_frame_unref(s->input_picture[0]->f);
1230
1231                     emms_c();
1232                     ff_vbv_update(s, 0);
1233
1234                     goto no_output_pic;
1235                 }
1236             }
1237
1238             if (s->flags & CODEC_FLAG_PASS2) {
1239                 for (i = 0; i < s->max_b_frames + 1; i++) {
1240                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1241
1242                     if (pict_num >= s->rc_context.num_entries)
1243                         break;
1244                     if (!s->input_picture[i]) {
1245                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1246                         break;
1247                     }
1248
1249                     s->input_picture[i]->f->pict_type =
1250                         s->rc_context.entry[pict_num].new_pict_type;
1251                 }
1252             }
1253
1254             if (s->avctx->b_frame_strategy == 0) {
1255                 b_frames = s->max_b_frames;
1256                 while (b_frames && !s->input_picture[b_frames])
1257                     b_frames--;
1258             } else if (s->avctx->b_frame_strategy == 1) {
1259                 for (i = 1; i < s->max_b_frames + 1; i++) {
1260                     if (s->input_picture[i] &&
1261                         s->input_picture[i]->b_frame_score == 0) {
1262                         s->input_picture[i]->b_frame_score =
1263                             get_intra_count(s,
1264                                             s->input_picture[i    ]->f->data[0],
1265                                             s->input_picture[i - 1]->f->data[0],
1266                                             s->linesize) + 1;
1267                     }
1268                 }
1269                 for (i = 0; i < s->max_b_frames + 1; i++) {
1270                     if (s->input_picture[i] == NULL ||
1271                         s->input_picture[i]->b_frame_score - 1 >
1272                             s->mb_num / s->avctx->b_sensitivity)
1273                         break;
1274                 }
1275
1276                 b_frames = FFMAX(0, i - 1);
1277
1278                 /* reset scores */
1279                 for (i = 0; i < b_frames + 1; i++) {
1280                     s->input_picture[i]->b_frame_score = 0;
1281                 }
1282             } else if (s->avctx->b_frame_strategy == 2) {
1283                 b_frames = estimate_best_b_count(s);
1284             } else {
1285                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1286                 b_frames = 0;
1287             }
1288
1289             emms_c();
1290
1291             for (i = b_frames - 1; i >= 0; i--) {
1292                 int type = s->input_picture[i]->f->pict_type;
1293                 if (type && type != AV_PICTURE_TYPE_B)
1294                     b_frames = i;
1295             }
1296             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1297                 b_frames == s->max_b_frames) {
1298                 av_log(s->avctx, AV_LOG_ERROR,
1299                        "warning, too many b frames in a row\n");
1300             }
1301
1302             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1303                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1304                     s->gop_size > s->picture_in_gop_number) {
1305                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1306                 } else {
1307                     if (s->flags & CODEC_FLAG_CLOSED_GOP)
1308                         b_frames = 0;
1309                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1310                 }
1311             }
1312
1313             if ((s->flags & CODEC_FLAG_CLOSED_GOP) && b_frames &&
1314                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1315                 b_frames--;
1316
1317             s->reordered_input_picture[0] = s->input_picture[b_frames];
1318             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1319                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1320             s->reordered_input_picture[0]->f->coded_picture_number =
1321                 s->coded_picture_number++;
1322             for (i = 0; i < b_frames; i++) {
1323                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1324                 s->reordered_input_picture[i + 1]->f->pict_type =
1325                     AV_PICTURE_TYPE_B;
1326                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1327                     s->coded_picture_number++;
1328             }
1329         }
1330     }
1331 no_output_pic:
1332     if (s->reordered_input_picture[0]) {
1333         s->reordered_input_picture[0]->reference =
1334            s->reordered_input_picture[0]->f->pict_type !=
1335                AV_PICTURE_TYPE_B ? 3 : 0;
1336
1337         ff_mpeg_unref_picture(s, &s->new_picture);
1338         if ((ret = ff_mpeg_ref_picture(s, &s->new_picture, s->reordered_input_picture[0])))
1339             return ret;
1340
1341         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1342             // input is a shared pix, so we can't modifiy it -> alloc a new
1343             // one & ensure that the shared one is reuseable
1344
1345             Picture *pic;
1346             int i = ff_find_unused_picture(s, 0);
1347             if (i < 0)
1348                 return i;
1349             pic = &s->picture[i];
1350
1351             pic->reference = s->reordered_input_picture[0]->reference;
1352             if (ff_alloc_picture(s, pic, 0) < 0) {
1353                 return -1;
1354             }
1355
1356             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1357             if (ret < 0)
1358                 return ret;
1359
1360             /* mark us unused / free shared pic */
1361             av_frame_unref(s->reordered_input_picture[0]->f);
1362             s->reordered_input_picture[0]->shared = 0;
1363
1364             s->current_picture_ptr = pic;
1365         } else {
1366             // input is not a shared pix -> reuse buffer for current_pix
1367             s->current_picture_ptr = s->reordered_input_picture[0];
1368             for (i = 0; i < 4; i++) {
1369                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1370             }
1371         }
1372         ff_mpeg_unref_picture(s, &s->current_picture);
1373         if ((ret = ff_mpeg_ref_picture(s, &s->current_picture,
1374                                        s->current_picture_ptr)) < 0)
1375             return ret;
1376
1377         s->picture_number = s->new_picture.f->display_picture_number;
1378     } else {
1379         ff_mpeg_unref_picture(s, &s->new_picture);
1380     }
1381     return 0;
1382 }
1383
1384 static void frame_end(MpegEncContext *s)
1385 {
1386     int i;
1387
1388     if (s->unrestricted_mv &&
1389         s->current_picture.reference &&
1390         !s->intra_only) {
1391         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1392         int hshift = desc->log2_chroma_w;
1393         int vshift = desc->log2_chroma_h;
1394         s->mpvencdsp.draw_edges(s->current_picture.f->data[0], s->linesize,
1395                                 s->h_edge_pos, s->v_edge_pos,
1396                                 EDGE_WIDTH, EDGE_WIDTH,
1397                                 EDGE_TOP | EDGE_BOTTOM);
1398         s->mpvencdsp.draw_edges(s->current_picture.f->data[1], s->uvlinesize,
1399                                 s->h_edge_pos >> hshift,
1400                                 s->v_edge_pos >> vshift,
1401                                 EDGE_WIDTH >> hshift,
1402                                 EDGE_WIDTH >> vshift,
1403                                 EDGE_TOP | EDGE_BOTTOM);
1404         s->mpvencdsp.draw_edges(s->current_picture.f->data[2], s->uvlinesize,
1405                                 s->h_edge_pos >> hshift,
1406                                 s->v_edge_pos >> vshift,
1407                                 EDGE_WIDTH >> hshift,
1408                                 EDGE_WIDTH >> vshift,
1409                                 EDGE_TOP | EDGE_BOTTOM);
1410     }
1411
1412     emms_c();
1413
1414     s->last_pict_type                 = s->pict_type;
1415     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1416     if (s->pict_type!= AV_PICTURE_TYPE_B)
1417         s->last_non_b_pict_type = s->pict_type;
1418
1419     if (s->encoding) {
1420         /* release non-reference frames */
1421         for (i = 0; i < MAX_PICTURE_COUNT; i++) {
1422             if (!s->picture[i].reference)
1423                 ff_mpeg_unref_picture(s, &s->picture[i]);
1424         }
1425     }
1426
1427     s->avctx->coded_frame = s->current_picture_ptr->f;
1428
1429 }
1430
1431 static void update_noise_reduction(MpegEncContext *s)
1432 {
1433     int intra, i;
1434
1435     for (intra = 0; intra < 2; intra++) {
1436         if (s->dct_count[intra] > (1 << 16)) {
1437             for (i = 0; i < 64; i++) {
1438                 s->dct_error_sum[intra][i] >>= 1;
1439             }
1440             s->dct_count[intra] >>= 1;
1441         }
1442
1443         for (i = 0; i < 64; i++) {
1444             s->dct_offset[intra][i] = (s->avctx->noise_reduction *
1445                                        s->dct_count[intra] +
1446                                        s->dct_error_sum[intra][i] / 2) /
1447                                       (s->dct_error_sum[intra][i] + 1);
1448         }
1449     }
1450 }
1451
1452 static int frame_start(MpegEncContext *s)
1453 {
1454     int ret;
1455
1456     /* mark & release old frames */
1457     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1458         s->last_picture_ptr != s->next_picture_ptr &&
1459         s->last_picture_ptr->f->buf[0]) {
1460         ff_mpeg_unref_picture(s, s->last_picture_ptr);
1461     }
1462
1463     s->current_picture_ptr->f->pict_type = s->pict_type;
1464     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1465
1466     ff_mpeg_unref_picture(s, &s->current_picture);
1467     if ((ret = ff_mpeg_ref_picture(s, &s->current_picture,
1468                                    s->current_picture_ptr)) < 0)
1469         return ret;
1470
1471     if (s->pict_type != AV_PICTURE_TYPE_B) {
1472         s->last_picture_ptr = s->next_picture_ptr;
1473         if (!s->droppable)
1474             s->next_picture_ptr = s->current_picture_ptr;
1475     }
1476
1477     if (s->last_picture_ptr) {
1478         ff_mpeg_unref_picture(s, &s->last_picture);
1479         if (s->last_picture_ptr->f->buf[0] &&
1480             (ret = ff_mpeg_ref_picture(s, &s->last_picture,
1481                                        s->last_picture_ptr)) < 0)
1482             return ret;
1483     }
1484     if (s->next_picture_ptr) {
1485         ff_mpeg_unref_picture(s, &s->next_picture);
1486         if (s->next_picture_ptr->f->buf[0] &&
1487             (ret = ff_mpeg_ref_picture(s, &s->next_picture,
1488                                        s->next_picture_ptr)) < 0)
1489             return ret;
1490     }
1491
1492     if (s->picture_structure!= PICT_FRAME) {
1493         int i;
1494         for (i = 0; i < 4; i++) {
1495             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1496                 s->current_picture.f->data[i] +=
1497                     s->current_picture.f->linesize[i];
1498             }
1499             s->current_picture.f->linesize[i] *= 2;
1500             s->last_picture.f->linesize[i]    *= 2;
1501             s->next_picture.f->linesize[i]    *= 2;
1502         }
1503     }
1504
1505     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1506         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1507         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1508     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1509         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1510         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1511     } else {
1512         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1513         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1514     }
1515
1516     if (s->dct_error_sum) {
1517         assert(s->avctx->noise_reduction && s->encoding);
1518         update_noise_reduction(s);
1519     }
1520
1521     return 0;
1522 }
1523
1524 int ff_MPV_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1525                           const AVFrame *pic_arg, int *got_packet)
1526 {
1527     MpegEncContext *s = avctx->priv_data;
1528     int i, stuffing_count, ret;
1529     int context_count = s->slice_context_count;
1530
1531     s->picture_in_gop_number++;
1532
1533     if (load_input_picture(s, pic_arg) < 0)
1534         return -1;
1535
1536     if (select_input_picture(s) < 0) {
1537         return -1;
1538     }
1539
1540     /* output? */
1541     if (s->new_picture.f->data[0]) {
1542         if (!pkt->data &&
1543             (ret = ff_alloc_packet(pkt, s->mb_width*s->mb_height*MAX_MB_BYTES)) < 0)
1544             return ret;
1545         if (s->mb_info) {
1546             s->mb_info_ptr = av_packet_new_side_data(pkt,
1547                                  AV_PKT_DATA_H263_MB_INFO,
1548                                  s->mb_width*s->mb_height*12);
1549             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1550         }
1551
1552         for (i = 0; i < context_count; i++) {
1553             int start_y = s->thread_context[i]->start_mb_y;
1554             int   end_y = s->thread_context[i]->  end_mb_y;
1555             int h       = s->mb_height;
1556             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1557             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1558
1559             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1560         }
1561
1562         s->pict_type = s->new_picture.f->pict_type;
1563         //emms_c();
1564         ret = frame_start(s);
1565         if (ret < 0)
1566             return ret;
1567 vbv_retry:
1568         if (encode_picture(s, s->picture_number) < 0)
1569             return -1;
1570
1571         avctx->header_bits = s->header_bits;
1572         avctx->mv_bits     = s->mv_bits;
1573         avctx->misc_bits   = s->misc_bits;
1574         avctx->i_tex_bits  = s->i_tex_bits;
1575         avctx->p_tex_bits  = s->p_tex_bits;
1576         avctx->i_count     = s->i_count;
1577         // FIXME f/b_count in avctx
1578         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1579         avctx->skip_count  = s->skip_count;
1580
1581         frame_end(s);
1582
1583         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1584             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1585
1586         if (avctx->rc_buffer_size) {
1587             RateControlContext *rcc = &s->rc_context;
1588             int max_size = rcc->buffer_index * avctx->rc_max_available_vbv_use;
1589
1590             if (put_bits_count(&s->pb) > max_size &&
1591                 s->lambda < s->avctx->lmax) {
1592                 s->next_lambda = FFMAX(s->lambda + 1, s->lambda *
1593                                        (s->qscale + 1) / s->qscale);
1594                 if (s->adaptive_quant) {
1595                     int i;
1596                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1597                         s->lambda_table[i] =
1598                             FFMAX(s->lambda_table[i] + 1,
1599                                   s->lambda_table[i] * (s->qscale + 1) /
1600                                   s->qscale);
1601                 }
1602                 s->mb_skipped = 0;        // done in frame_start()
1603                 // done in encode_picture() so we must undo it
1604                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1605                     if (s->flipflop_rounding          ||
1606                         s->codec_id == AV_CODEC_ID_H263P ||
1607                         s->codec_id == AV_CODEC_ID_MPEG4)
1608                         s->no_rounding ^= 1;
1609                 }
1610                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1611                     s->time_base       = s->last_time_base;
1612                     s->last_non_b_time = s->time - s->pp_time;
1613                 }
1614                 for (i = 0; i < context_count; i++) {
1615                     PutBitContext *pb = &s->thread_context[i]->pb;
1616                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1617                 }
1618                 goto vbv_retry;
1619             }
1620
1621             assert(s->avctx->rc_max_rate);
1622         }
1623
1624         if (s->flags & CODEC_FLAG_PASS1)
1625             ff_write_pass1_stats(s);
1626
1627         for (i = 0; i < 4; i++) {
1628             s->current_picture_ptr->f->error[i] = s->current_picture.f->error[i];
1629             avctx->error[i] += s->current_picture_ptr->f->error[i];
1630         }
1631
1632         if (s->flags & CODEC_FLAG_PASS1)
1633             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits +
1634                    avctx->i_tex_bits + avctx->p_tex_bits ==
1635                        put_bits_count(&s->pb));
1636         flush_put_bits(&s->pb);
1637         s->frame_bits  = put_bits_count(&s->pb);
1638
1639         stuffing_count = ff_vbv_update(s, s->frame_bits);
1640         if (stuffing_count) {
1641             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1642                     stuffing_count + 50) {
1643                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1644                 return -1;
1645             }
1646
1647             switch (s->codec_id) {
1648             case AV_CODEC_ID_MPEG1VIDEO:
1649             case AV_CODEC_ID_MPEG2VIDEO:
1650                 while (stuffing_count--) {
1651                     put_bits(&s->pb, 8, 0);
1652                 }
1653             break;
1654             case AV_CODEC_ID_MPEG4:
1655                 put_bits(&s->pb, 16, 0);
1656                 put_bits(&s->pb, 16, 0x1C3);
1657                 stuffing_count -= 4;
1658                 while (stuffing_count--) {
1659                     put_bits(&s->pb, 8, 0xFF);
1660                 }
1661             break;
1662             default:
1663                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1664             }
1665             flush_put_bits(&s->pb);
1666             s->frame_bits  = put_bits_count(&s->pb);
1667         }
1668
1669         /* update mpeg1/2 vbv_delay for CBR */
1670         if (s->avctx->rc_max_rate                          &&
1671             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1672             s->out_format == FMT_MPEG1                     &&
1673             90000LL * (avctx->rc_buffer_size - 1) <=
1674                 s->avctx->rc_max_rate * 0xFFFFLL) {
1675             int vbv_delay, min_delay;
1676             double inbits  = s->avctx->rc_max_rate *
1677                              av_q2d(s->avctx->time_base);
1678             int    minbits = s->frame_bits - 8 *
1679                              (s->vbv_delay_ptr - s->pb.buf - 1);
1680             double bits    = s->rc_context.buffer_index + minbits - inbits;
1681
1682             if (bits < 0)
1683                 av_log(s->avctx, AV_LOG_ERROR,
1684                        "Internal error, negative bits\n");
1685
1686             assert(s->repeat_first_field == 0);
1687
1688             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
1689             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
1690                         s->avctx->rc_max_rate;
1691
1692             vbv_delay = FFMAX(vbv_delay, min_delay);
1693
1694             assert(vbv_delay < 0xFFFF);
1695
1696             s->vbv_delay_ptr[0] &= 0xF8;
1697             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
1698             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
1699             s->vbv_delay_ptr[2] &= 0x07;
1700             s->vbv_delay_ptr[2] |= vbv_delay << 3;
1701             avctx->vbv_delay     = vbv_delay * 300;
1702         }
1703         s->total_bits     += s->frame_bits;
1704         avctx->frame_bits  = s->frame_bits;
1705
1706         pkt->pts = s->current_picture.f->pts;
1707         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
1708             if (!s->current_picture.f->coded_picture_number)
1709                 pkt->dts = pkt->pts - s->dts_delta;
1710             else
1711                 pkt->dts = s->reordered_pts;
1712             s->reordered_pts = pkt->pts;
1713         } else
1714             pkt->dts = pkt->pts;
1715         if (s->current_picture.f->key_frame)
1716             pkt->flags |= AV_PKT_FLAG_KEY;
1717         if (s->mb_info)
1718             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
1719     } else {
1720         s->frame_bits = 0;
1721     }
1722     assert((s->frame_bits & 7) == 0);
1723
1724     pkt->size = s->frame_bits / 8;
1725     *got_packet = !!pkt->size;
1726     return 0;
1727 }
1728
1729 static inline void dct_single_coeff_elimination(MpegEncContext *s,
1730                                                 int n, int threshold)
1731 {
1732     static const char tab[64] = {
1733         3, 2, 2, 1, 1, 1, 1, 1,
1734         1, 1, 1, 1, 1, 1, 1, 1,
1735         1, 1, 1, 1, 1, 1, 1, 1,
1736         0, 0, 0, 0, 0, 0, 0, 0,
1737         0, 0, 0, 0, 0, 0, 0, 0,
1738         0, 0, 0, 0, 0, 0, 0, 0,
1739         0, 0, 0, 0, 0, 0, 0, 0,
1740         0, 0, 0, 0, 0, 0, 0, 0
1741     };
1742     int score = 0;
1743     int run = 0;
1744     int i;
1745     int16_t *block = s->block[n];
1746     const int last_index = s->block_last_index[n];
1747     int skip_dc;
1748
1749     if (threshold < 0) {
1750         skip_dc = 0;
1751         threshold = -threshold;
1752     } else
1753         skip_dc = 1;
1754
1755     /* Are all we could set to zero already zero? */
1756     if (last_index <= skip_dc - 1)
1757         return;
1758
1759     for (i = 0; i <= last_index; i++) {
1760         const int j = s->intra_scantable.permutated[i];
1761         const int level = FFABS(block[j]);
1762         if (level == 1) {
1763             if (skip_dc && i == 0)
1764                 continue;
1765             score += tab[run];
1766             run = 0;
1767         } else if (level > 1) {
1768             return;
1769         } else {
1770             run++;
1771         }
1772     }
1773     if (score >= threshold)
1774         return;
1775     for (i = skip_dc; i <= last_index; i++) {
1776         const int j = s->intra_scantable.permutated[i];
1777         block[j] = 0;
1778     }
1779     if (block[0])
1780         s->block_last_index[n] = 0;
1781     else
1782         s->block_last_index[n] = -1;
1783 }
1784
1785 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
1786                                int last_index)
1787 {
1788     int i;
1789     const int maxlevel = s->max_qcoeff;
1790     const int minlevel = s->min_qcoeff;
1791     int overflow = 0;
1792
1793     if (s->mb_intra) {
1794         i = 1; // skip clipping of intra dc
1795     } else
1796         i = 0;
1797
1798     for (; i <= last_index; i++) {
1799         const int j = s->intra_scantable.permutated[i];
1800         int level = block[j];
1801
1802         if (level > maxlevel) {
1803             level = maxlevel;
1804             overflow++;
1805         } else if (level < minlevel) {
1806             level = minlevel;
1807             overflow++;
1808         }
1809
1810         block[j] = level;
1811     }
1812
1813     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1814         av_log(s->avctx, AV_LOG_INFO,
1815                "warning, clipping %d dct coefficients to %d..%d\n",
1816                overflow, minlevel, maxlevel);
1817 }
1818
1819 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
1820 {
1821     int x, y;
1822     // FIXME optimize
1823     for (y = 0; y < 8; y++) {
1824         for (x = 0; x < 8; x++) {
1825             int x2, y2;
1826             int sum = 0;
1827             int sqr = 0;
1828             int count = 0;
1829
1830             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
1831                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
1832                     int v = ptr[x2 + y2 * stride];
1833                     sum += v;
1834                     sqr += v * v;
1835                     count++;
1836                 }
1837             }
1838             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
1839         }
1840     }
1841 }
1842
1843 static av_always_inline void encode_mb_internal(MpegEncContext *s,
1844                                                 int motion_x, int motion_y,
1845                                                 int mb_block_height,
1846                                                 int mb_block_count)
1847 {
1848     int16_t weight[8][64];
1849     int16_t orig[8][64];
1850     const int mb_x = s->mb_x;
1851     const int mb_y = s->mb_y;
1852     int i;
1853     int skip_dct[8];
1854     int dct_offset = s->linesize * 8; // default for progressive frames
1855     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1856     ptrdiff_t wrap_y, wrap_c;
1857
1858     for (i = 0; i < mb_block_count; i++)
1859         skip_dct[i] = s->skipdct;
1860
1861     if (s->adaptive_quant) {
1862         const int last_qp = s->qscale;
1863         const int mb_xy = mb_x + mb_y * s->mb_stride;
1864
1865         s->lambda = s->lambda_table[mb_xy];
1866         update_qscale(s);
1867
1868         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
1869             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
1870             s->dquant = s->qscale - last_qp;
1871
1872             if (s->out_format == FMT_H263) {
1873                 s->dquant = av_clip(s->dquant, -2, 2);
1874
1875                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
1876                     if (!s->mb_intra) {
1877                         if (s->pict_type == AV_PICTURE_TYPE_B) {
1878                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
1879                                 s->dquant = 0;
1880                         }
1881                         if (s->mv_type == MV_TYPE_8X8)
1882                             s->dquant = 0;
1883                     }
1884                 }
1885             }
1886         }
1887         ff_set_qscale(s, last_qp + s->dquant);
1888     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
1889         ff_set_qscale(s, s->qscale + s->dquant);
1890
1891     wrap_y = s->linesize;
1892     wrap_c = s->uvlinesize;
1893     ptr_y  = s->new_picture.f->data[0] +
1894              (mb_y * 16 * wrap_y)              + mb_x * 16;
1895     ptr_cb = s->new_picture.f->data[1] +
1896              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1897     ptr_cr = s->new_picture.f->data[2] +
1898              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1899
1900     if (mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) {
1901         uint8_t *ebuf = s->edge_emu_buffer + 32;
1902         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
1903                                  wrap_y, wrap_y,
1904                                  16, 16, mb_x * 16, mb_y * 16,
1905                                  s->width, s->height);
1906         ptr_y = ebuf;
1907         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y, ptr_cb,
1908                                  wrap_c, wrap_c,
1909                                  8, mb_block_height, mb_x * 8, mb_y * 8,
1910                                  s->width >> 1, s->height >> 1);
1911         ptr_cb = ebuf + 18 * wrap_y;
1912         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y + 8, ptr_cr,
1913                                  wrap_c, wrap_c,
1914                                  8, mb_block_height, mb_x * 8, mb_y * 8,
1915                                  s->width >> 1, s->height >> 1);
1916         ptr_cr = ebuf + 18 * wrap_y + 8;
1917     }
1918
1919     if (s->mb_intra) {
1920         if (s->flags & CODEC_FLAG_INTERLACED_DCT) {
1921             int progressive_score, interlaced_score;
1922
1923             s->interlaced_dct = 0;
1924             progressive_score = s->dsp.ildct_cmp[4](s, ptr_y,
1925                                                     NULL, wrap_y, 8) +
1926                                 s->dsp.ildct_cmp[4](s, ptr_y + wrap_y * 8,
1927                                                     NULL, wrap_y, 8) - 400;
1928
1929             if (progressive_score > 0) {
1930                 interlaced_score = s->dsp.ildct_cmp[4](s, ptr_y,
1931                                                        NULL, wrap_y * 2, 8) +
1932                                    s->dsp.ildct_cmp[4](s, ptr_y + wrap_y,
1933                                                        NULL, wrap_y * 2, 8);
1934                 if (progressive_score > interlaced_score) {
1935                     s->interlaced_dct = 1;
1936
1937                     dct_offset = wrap_y;
1938                     wrap_y <<= 1;
1939                     if (s->chroma_format == CHROMA_422)
1940                         wrap_c <<= 1;
1941                 }
1942             }
1943         }
1944
1945         s->dsp.get_pixels(s->block[0], ptr_y                  , wrap_y);
1946         s->dsp.get_pixels(s->block[1], ptr_y              + 8 , wrap_y);
1947         s->dsp.get_pixels(s->block[2], ptr_y + dct_offset     , wrap_y);
1948         s->dsp.get_pixels(s->block[3], ptr_y + dct_offset + 8 , wrap_y);
1949
1950         if (s->flags & CODEC_FLAG_GRAY) {
1951             skip_dct[4] = 1;
1952             skip_dct[5] = 1;
1953         } else {
1954             s->dsp.get_pixels(s->block[4], ptr_cb, wrap_c);
1955             s->dsp.get_pixels(s->block[5], ptr_cr, wrap_c);
1956             if (!s->chroma_y_shift) { /* 422 */
1957                 s->dsp.get_pixels(s->block[6],
1958                                   ptr_cb + (dct_offset >> 1), wrap_c);
1959                 s->dsp.get_pixels(s->block[7],
1960                                   ptr_cr + (dct_offset >> 1), wrap_c);
1961             }
1962         }
1963     } else {
1964         op_pixels_func (*op_pix)[4];
1965         qpel_mc_func (*op_qpix)[16];
1966         uint8_t *dest_y, *dest_cb, *dest_cr;
1967
1968         dest_y  = s->dest[0];
1969         dest_cb = s->dest[1];
1970         dest_cr = s->dest[2];
1971
1972         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
1973             op_pix  = s->hdsp.put_pixels_tab;
1974             op_qpix = s->qdsp.put_qpel_pixels_tab;
1975         } else {
1976             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
1977             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
1978         }
1979
1980         if (s->mv_dir & MV_DIR_FORWARD) {
1981             ff_MPV_motion(s, dest_y, dest_cb, dest_cr, 0,
1982                           s->last_picture.f->data,
1983                           op_pix, op_qpix);
1984             op_pix  = s->hdsp.avg_pixels_tab;
1985             op_qpix = s->qdsp.avg_qpel_pixels_tab;
1986         }
1987         if (s->mv_dir & MV_DIR_BACKWARD) {
1988             ff_MPV_motion(s, dest_y, dest_cb, dest_cr, 1,
1989                           s->next_picture.f->data,
1990                           op_pix, op_qpix);
1991         }
1992
1993         if (s->flags & CODEC_FLAG_INTERLACED_DCT) {
1994             int progressive_score, interlaced_score;
1995
1996             s->interlaced_dct = 0;
1997             progressive_score = s->dsp.ildct_cmp[0](s, dest_y,
1998                                                     ptr_y,              wrap_y,
1999                                                     8) +
2000                                 s->dsp.ildct_cmp[0](s, dest_y + wrap_y * 8,
2001                                                     ptr_y + wrap_y * 8, wrap_y,
2002                                                     8) - 400;
2003
2004             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2005                 progressive_score -= 400;
2006
2007             if (progressive_score > 0) {
2008                 interlaced_score = s->dsp.ildct_cmp[0](s, dest_y,
2009                                                        ptr_y,
2010                                                        wrap_y * 2, 8) +
2011                                    s->dsp.ildct_cmp[0](s, dest_y + wrap_y,
2012                                                        ptr_y + wrap_y,
2013                                                        wrap_y * 2, 8);
2014
2015                 if (progressive_score > interlaced_score) {
2016                     s->interlaced_dct = 1;
2017
2018                     dct_offset = wrap_y;
2019                     wrap_y <<= 1;
2020                     if (s->chroma_format == CHROMA_422)
2021                         wrap_c <<= 1;
2022                 }
2023             }
2024         }
2025
2026         s->dsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2027         s->dsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2028         s->dsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2029                            dest_y + dct_offset, wrap_y);
2030         s->dsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2031                            dest_y + dct_offset + 8, wrap_y);
2032
2033         if (s->flags & CODEC_FLAG_GRAY) {
2034             skip_dct[4] = 1;
2035             skip_dct[5] = 1;
2036         } else {
2037             s->dsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2038             s->dsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2039             if (!s->chroma_y_shift) { /* 422 */
2040                 s->dsp.diff_pixels(s->block[6], ptr_cb + (dct_offset >> 1),
2041                                    dest_cb + (dct_offset >> 1), wrap_c);
2042                 s->dsp.diff_pixels(s->block[7], ptr_cr + (dct_offset >> 1),
2043                                    dest_cr + (dct_offset >> 1), wrap_c);
2044             }
2045         }
2046         /* pre quantization */
2047         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2048                 2 * s->qscale * s->qscale) {
2049             // FIXME optimize
2050             if (s->dsp.sad[1](NULL, ptr_y , dest_y,
2051                               wrap_y, 8) < 20 * s->qscale)
2052                 skip_dct[0] = 1;
2053             if (s->dsp.sad[1](NULL, ptr_y + 8,
2054                               dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2055                 skip_dct[1] = 1;
2056             if (s->dsp.sad[1](NULL, ptr_y + dct_offset,
2057                               dest_y + dct_offset, wrap_y, 8) < 20 * s->qscale)
2058                 skip_dct[2] = 1;
2059             if (s->dsp.sad[1](NULL, ptr_y + dct_offset + 8,
2060                               dest_y + dct_offset + 8,
2061                               wrap_y, 8) < 20 * s->qscale)
2062                 skip_dct[3] = 1;
2063             if (s->dsp.sad[1](NULL, ptr_cb, dest_cb,
2064                               wrap_c, 8) < 20 * s->qscale)
2065                 skip_dct[4] = 1;
2066             if (s->dsp.sad[1](NULL, ptr_cr, dest_cr,
2067                               wrap_c, 8) < 20 * s->qscale)
2068                 skip_dct[5] = 1;
2069             if (!s->chroma_y_shift) { /* 422 */
2070                 if (s->dsp.sad[1](NULL, ptr_cb + (dct_offset >> 1),
2071                                   dest_cb + (dct_offset >> 1),
2072                                   wrap_c, 8) < 20 * s->qscale)
2073                     skip_dct[6] = 1;
2074                 if (s->dsp.sad[1](NULL, ptr_cr + (dct_offset >> 1),
2075                                   dest_cr + (dct_offset >> 1),
2076                                   wrap_c, 8) < 20 * s->qscale)
2077                     skip_dct[7] = 1;
2078             }
2079         }
2080     }
2081
2082     if (s->quantizer_noise_shaping) {
2083         if (!skip_dct[0])
2084             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2085         if (!skip_dct[1])
2086             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2087         if (!skip_dct[2])
2088             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2089         if (!skip_dct[3])
2090             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2091         if (!skip_dct[4])
2092             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2093         if (!skip_dct[5])
2094             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2095         if (!s->chroma_y_shift) { /* 422 */
2096             if (!skip_dct[6])
2097                 get_visual_weight(weight[6], ptr_cb + (dct_offset >> 1),
2098                                   wrap_c);
2099             if (!skip_dct[7])
2100                 get_visual_weight(weight[7], ptr_cr + (dct_offset >> 1),
2101                                   wrap_c);
2102         }
2103         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2104     }
2105
2106     /* DCT & quantize */
2107     assert(s->out_format != FMT_MJPEG || s->qscale == 8);
2108     {
2109         for (i = 0; i < mb_block_count; i++) {
2110             if (!skip_dct[i]) {
2111                 int overflow;
2112                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2113                 // FIXME we could decide to change to quantizer instead of
2114                 // clipping
2115                 // JS: I don't think that would be a good idea it could lower
2116                 //     quality instead of improve it. Just INTRADC clipping
2117                 //     deserves changes in quantizer
2118                 if (overflow)
2119                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2120             } else
2121                 s->block_last_index[i] = -1;
2122         }
2123         if (s->quantizer_noise_shaping) {
2124             for (i = 0; i < mb_block_count; i++) {
2125                 if (!skip_dct[i]) {
2126                     s->block_last_index[i] =
2127                         dct_quantize_refine(s, s->block[i], weight[i],
2128                                             orig[i], i, s->qscale);
2129                 }
2130             }
2131         }
2132
2133         if (s->luma_elim_threshold && !s->mb_intra)
2134             for (i = 0; i < 4; i++)
2135                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2136         if (s->chroma_elim_threshold && !s->mb_intra)
2137             for (i = 4; i < mb_block_count; i++)
2138                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2139
2140         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2141             for (i = 0; i < mb_block_count; i++) {
2142                 if (s->block_last_index[i] == -1)
2143                     s->coded_score[i] = INT_MAX / 256;
2144             }
2145         }
2146     }
2147
2148     if ((s->flags & CODEC_FLAG_GRAY) && s->mb_intra) {
2149         s->block_last_index[4] =
2150         s->block_last_index[5] = 0;
2151         s->block[4][0] =
2152         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2153     }
2154
2155     // non c quantize code returns incorrect block_last_index FIXME
2156     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2157         for (i = 0; i < mb_block_count; i++) {
2158             int j;
2159             if (s->block_last_index[i] > 0) {
2160                 for (j = 63; j > 0; j--) {
2161                     if (s->block[i][s->intra_scantable.permutated[j]])
2162                         break;
2163                 }
2164                 s->block_last_index[i] = j;
2165             }
2166         }
2167     }
2168
2169     /* huffman encode */
2170     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2171     case AV_CODEC_ID_MPEG1VIDEO:
2172     case AV_CODEC_ID_MPEG2VIDEO:
2173         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2174             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2175         break;
2176     case AV_CODEC_ID_MPEG4:
2177         if (CONFIG_MPEG4_ENCODER)
2178             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2179         break;
2180     case AV_CODEC_ID_MSMPEG4V2:
2181     case AV_CODEC_ID_MSMPEG4V3:
2182     case AV_CODEC_ID_WMV1:
2183         if (CONFIG_MSMPEG4_ENCODER)
2184             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2185         break;
2186     case AV_CODEC_ID_WMV2:
2187         if (CONFIG_WMV2_ENCODER)
2188             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2189         break;
2190     case AV_CODEC_ID_H261:
2191         if (CONFIG_H261_ENCODER)
2192             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2193         break;
2194     case AV_CODEC_ID_H263:
2195     case AV_CODEC_ID_H263P:
2196     case AV_CODEC_ID_FLV1:
2197     case AV_CODEC_ID_RV10:
2198     case AV_CODEC_ID_RV20:
2199         if (CONFIG_H263_ENCODER)
2200             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2201         break;
2202     case AV_CODEC_ID_MJPEG:
2203         if (CONFIG_MJPEG_ENCODER)
2204             ff_mjpeg_encode_mb(s, s->block);
2205         break;
2206     default:
2207         assert(0);
2208     }
2209 }
2210
2211 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2212 {
2213     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
2214     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
2215 }
2216
2217 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2218     int i;
2219
2220     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2221
2222     /* mpeg1 */
2223     d->mb_skip_run= s->mb_skip_run;
2224     for(i=0; i<3; i++)
2225         d->last_dc[i] = s->last_dc[i];
2226
2227     /* statistics */
2228     d->mv_bits= s->mv_bits;
2229     d->i_tex_bits= s->i_tex_bits;
2230     d->p_tex_bits= s->p_tex_bits;
2231     d->i_count= s->i_count;
2232     d->f_count= s->f_count;
2233     d->b_count= s->b_count;
2234     d->skip_count= s->skip_count;
2235     d->misc_bits= s->misc_bits;
2236     d->last_bits= 0;
2237
2238     d->mb_skipped= 0;
2239     d->qscale= s->qscale;
2240     d->dquant= s->dquant;
2241
2242     d->esc3_level_length= s->esc3_level_length;
2243 }
2244
2245 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2246     int i;
2247
2248     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2249     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2250
2251     /* mpeg1 */
2252     d->mb_skip_run= s->mb_skip_run;
2253     for(i=0; i<3; i++)
2254         d->last_dc[i] = s->last_dc[i];
2255
2256     /* statistics */
2257     d->mv_bits= s->mv_bits;
2258     d->i_tex_bits= s->i_tex_bits;
2259     d->p_tex_bits= s->p_tex_bits;
2260     d->i_count= s->i_count;
2261     d->f_count= s->f_count;
2262     d->b_count= s->b_count;
2263     d->skip_count= s->skip_count;
2264     d->misc_bits= s->misc_bits;
2265
2266     d->mb_intra= s->mb_intra;
2267     d->mb_skipped= s->mb_skipped;
2268     d->mv_type= s->mv_type;
2269     d->mv_dir= s->mv_dir;
2270     d->pb= s->pb;
2271     if(s->data_partitioning){
2272         d->pb2= s->pb2;
2273         d->tex_pb= s->tex_pb;
2274     }
2275     d->block= s->block;
2276     for(i=0; i<8; i++)
2277         d->block_last_index[i]= s->block_last_index[i];
2278     d->interlaced_dct= s->interlaced_dct;
2279     d->qscale= s->qscale;
2280
2281     d->esc3_level_length= s->esc3_level_length;
2282 }
2283
2284 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2285                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2286                            int *dmin, int *next_block, int motion_x, int motion_y)
2287 {
2288     int score;
2289     uint8_t *dest_backup[3];
2290
2291     copy_context_before_encode(s, backup, type);
2292
2293     s->block= s->blocks[*next_block];
2294     s->pb= pb[*next_block];
2295     if(s->data_partitioning){
2296         s->pb2   = pb2   [*next_block];
2297         s->tex_pb= tex_pb[*next_block];
2298     }
2299
2300     if(*next_block){
2301         memcpy(dest_backup, s->dest, sizeof(s->dest));
2302         s->dest[0] = s->rd_scratchpad;
2303         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
2304         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
2305         assert(s->linesize >= 32); //FIXME
2306     }
2307
2308     encode_mb(s, motion_x, motion_y);
2309
2310     score= put_bits_count(&s->pb);
2311     if(s->data_partitioning){
2312         score+= put_bits_count(&s->pb2);
2313         score+= put_bits_count(&s->tex_pb);
2314     }
2315
2316     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2317         ff_MPV_decode_mb(s, s->block);
2318
2319         score *= s->lambda2;
2320         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2321     }
2322
2323     if(*next_block){
2324         memcpy(s->dest, dest_backup, sizeof(s->dest));
2325     }
2326
2327     if(score<*dmin){
2328         *dmin= score;
2329         *next_block^=1;
2330
2331         copy_context_after_encode(best, s, type);
2332     }
2333 }
2334
2335 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2336     uint32_t *sq = ff_square_tab + 256;
2337     int acc=0;
2338     int x,y;
2339
2340     if(w==16 && h==16)
2341         return s->dsp.sse[0](NULL, src1, src2, stride, 16);
2342     else if(w==8 && h==8)
2343         return s->dsp.sse[1](NULL, src1, src2, stride, 8);
2344
2345     for(y=0; y<h; y++){
2346         for(x=0; x<w; x++){
2347             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2348         }
2349     }
2350
2351     assert(acc>=0);
2352
2353     return acc;
2354 }
2355
2356 static int sse_mb(MpegEncContext *s){
2357     int w= 16;
2358     int h= 16;
2359
2360     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2361     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2362
2363     if(w==16 && h==16)
2364       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2365         return  s->dsp.nsse[0](s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2366                +s->dsp.nsse[1](s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2367                +s->dsp.nsse[1](s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2368       }else{
2369         return  s->dsp.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2370                +s->dsp.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2371                +s->dsp.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2372       }
2373     else
2374         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2375                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2376                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2377 }
2378
2379 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2380     MpegEncContext *s= *(void**)arg;
2381
2382
2383     s->me.pre_pass=1;
2384     s->me.dia_size= s->avctx->pre_dia_size;
2385     s->first_slice_line=1;
2386     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2387         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2388             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2389         }
2390         s->first_slice_line=0;
2391     }
2392
2393     s->me.pre_pass=0;
2394
2395     return 0;
2396 }
2397
2398 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2399     MpegEncContext *s= *(void**)arg;
2400
2401     s->me.dia_size= s->avctx->dia_size;
2402     s->first_slice_line=1;
2403     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2404         s->mb_x=0; //for block init below
2405         ff_init_block_index(s);
2406         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2407             s->block_index[0]+=2;
2408             s->block_index[1]+=2;
2409             s->block_index[2]+=2;
2410             s->block_index[3]+=2;
2411
2412             /* compute motion vector & mb_type and store in context */
2413             if(s->pict_type==AV_PICTURE_TYPE_B)
2414                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2415             else
2416                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2417         }
2418         s->first_slice_line=0;
2419     }
2420     return 0;
2421 }
2422
2423 static int mb_var_thread(AVCodecContext *c, void *arg){
2424     MpegEncContext *s= *(void**)arg;
2425     int mb_x, mb_y;
2426
2427     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2428         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2429             int xx = mb_x * 16;
2430             int yy = mb_y * 16;
2431             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2432             int varc;
2433             int sum = s->mpvencdsp.pix_sum(pix, s->linesize);
2434
2435             varc = (s->mpvencdsp.pix_norm1(pix, s->linesize) -
2436                     (((unsigned) sum * sum) >> 8) + 500 + 128) >> 8;
2437
2438             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2439             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2440             s->me.mb_var_sum_temp    += varc;
2441         }
2442     }
2443     return 0;
2444 }
2445
2446 static void write_slice_end(MpegEncContext *s){
2447     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2448         if(s->partitioned_frame){
2449             ff_mpeg4_merge_partitions(s);
2450         }
2451
2452         ff_mpeg4_stuffing(&s->pb);
2453     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2454         ff_mjpeg_encode_stuffing(&s->pb);
2455     }
2456
2457     avpriv_align_put_bits(&s->pb);
2458     flush_put_bits(&s->pb);
2459
2460     if((s->flags&CODEC_FLAG_PASS1) && !s->partitioned_frame)
2461         s->misc_bits+= get_bits_diff(s);
2462 }
2463
2464 static void write_mb_info(MpegEncContext *s)
2465 {
2466     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2467     int offset = put_bits_count(&s->pb);
2468     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2469     int gobn = s->mb_y / s->gob_index;
2470     int pred_x, pred_y;
2471     if (CONFIG_H263_ENCODER)
2472         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2473     bytestream_put_le32(&ptr, offset);
2474     bytestream_put_byte(&ptr, s->qscale);
2475     bytestream_put_byte(&ptr, gobn);
2476     bytestream_put_le16(&ptr, mba);
2477     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2478     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2479     /* 4MV not implemented */
2480     bytestream_put_byte(&ptr, 0); /* hmv2 */
2481     bytestream_put_byte(&ptr, 0); /* vmv2 */
2482 }
2483
2484 static void update_mb_info(MpegEncContext *s, int startcode)
2485 {
2486     if (!s->mb_info)
2487         return;
2488     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2489         s->mb_info_size += 12;
2490         s->prev_mb_info = s->last_mb_info;
2491     }
2492     if (startcode) {
2493         s->prev_mb_info = put_bits_count(&s->pb)/8;
2494         /* This might have incremented mb_info_size above, and we return without
2495          * actually writing any info into that slot yet. But in that case,
2496          * this will be called again at the start of the after writing the
2497          * start code, actually writing the mb info. */
2498         return;
2499     }
2500
2501     s->last_mb_info = put_bits_count(&s->pb)/8;
2502     if (!s->mb_info_size)
2503         s->mb_info_size += 12;
2504     write_mb_info(s);
2505 }
2506
2507 static int encode_thread(AVCodecContext *c, void *arg){
2508     MpegEncContext *s= *(void**)arg;
2509     int mb_x, mb_y, pdif = 0;
2510     int chr_h= 16>>s->chroma_y_shift;
2511     int i, j;
2512     MpegEncContext best_s, backup_s;
2513     uint8_t bit_buf[2][MAX_MB_BYTES];
2514     uint8_t bit_buf2[2][MAX_MB_BYTES];
2515     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2516     PutBitContext pb[2], pb2[2], tex_pb[2];
2517
2518     for(i=0; i<2; i++){
2519         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2520         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2521         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2522     }
2523
2524     s->last_bits= put_bits_count(&s->pb);
2525     s->mv_bits=0;
2526     s->misc_bits=0;
2527     s->i_tex_bits=0;
2528     s->p_tex_bits=0;
2529     s->i_count=0;
2530     s->f_count=0;
2531     s->b_count=0;
2532     s->skip_count=0;
2533
2534     for(i=0; i<3; i++){
2535         /* init last dc values */
2536         /* note: quant matrix value (8) is implied here */
2537         s->last_dc[i] = 128 << s->intra_dc_precision;
2538
2539         s->current_picture.f->error[i] = 0;
2540     }
2541     s->mb_skip_run = 0;
2542     memset(s->last_mv, 0, sizeof(s->last_mv));
2543
2544     s->last_mv_dir = 0;
2545
2546     switch(s->codec_id){
2547     case AV_CODEC_ID_H263:
2548     case AV_CODEC_ID_H263P:
2549     case AV_CODEC_ID_FLV1:
2550         if (CONFIG_H263_ENCODER)
2551             s->gob_index = ff_h263_get_gob_height(s);
2552         break;
2553     case AV_CODEC_ID_MPEG4:
2554         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2555             ff_mpeg4_init_partitions(s);
2556         break;
2557     }
2558
2559     s->resync_mb_x=0;
2560     s->resync_mb_y=0;
2561     s->first_slice_line = 1;
2562     s->ptr_lastgob = s->pb.buf;
2563     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2564         s->mb_x=0;
2565         s->mb_y= mb_y;
2566
2567         ff_set_qscale(s, s->qscale);
2568         ff_init_block_index(s);
2569
2570         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2571             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2572             int mb_type= s->mb_type[xy];
2573 //            int d;
2574             int dmin= INT_MAX;
2575             int dir;
2576
2577             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2578                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2579                 return -1;
2580             }
2581             if(s->data_partitioning){
2582                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2583                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2584                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2585                     return -1;
2586                 }
2587             }
2588
2589             s->mb_x = mb_x;
2590             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2591             ff_update_block_index(s);
2592
2593             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
2594                 ff_h261_reorder_mb_index(s);
2595                 xy= s->mb_y*s->mb_stride + s->mb_x;
2596                 mb_type= s->mb_type[xy];
2597             }
2598
2599             /* write gob / video packet header  */
2600             if(s->rtp_mode){
2601                 int current_packet_size, is_gob_start;
2602
2603                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2604
2605                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2606
2607                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2608
2609                 switch(s->codec_id){
2610                 case AV_CODEC_ID_H263:
2611                 case AV_CODEC_ID_H263P:
2612                     if(!s->h263_slice_structured)
2613                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2614                     break;
2615                 case AV_CODEC_ID_MPEG2VIDEO:
2616                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2617                 case AV_CODEC_ID_MPEG1VIDEO:
2618                     if(s->mb_skip_run) is_gob_start=0;
2619                     break;
2620                 }
2621
2622                 if(is_gob_start){
2623                     if(s->start_mb_y != mb_y || mb_x!=0){
2624                         write_slice_end(s);
2625
2626                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
2627                             ff_mpeg4_init_partitions(s);
2628                         }
2629                     }
2630
2631                     assert((put_bits_count(&s->pb)&7) == 0);
2632                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2633
2634                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
2635                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2636                         int d = 100 / s->error_rate;
2637                         if(r % d == 0){
2638                             current_packet_size=0;
2639                             s->pb.buf_ptr= s->ptr_lastgob;
2640                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2641                         }
2642                     }
2643
2644                     if (s->avctx->rtp_callback){
2645                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2646                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2647                     }
2648                     update_mb_info(s, 1);
2649
2650                     switch(s->codec_id){
2651                     case AV_CODEC_ID_MPEG4:
2652                         if (CONFIG_MPEG4_ENCODER) {
2653                             ff_mpeg4_encode_video_packet_header(s);
2654                             ff_mpeg4_clean_buffers(s);
2655                         }
2656                     break;
2657                     case AV_CODEC_ID_MPEG1VIDEO:
2658                     case AV_CODEC_ID_MPEG2VIDEO:
2659                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2660                             ff_mpeg1_encode_slice_header(s);
2661                             ff_mpeg1_clean_buffers(s);
2662                         }
2663                     break;
2664                     case AV_CODEC_ID_H263:
2665                     case AV_CODEC_ID_H263P:
2666                         if (CONFIG_H263_ENCODER)
2667                             ff_h263_encode_gob_header(s, mb_y);
2668                     break;
2669                     }
2670
2671                     if(s->flags&CODEC_FLAG_PASS1){
2672                         int bits= put_bits_count(&s->pb);
2673                         s->misc_bits+= bits - s->last_bits;
2674                         s->last_bits= bits;
2675                     }
2676
2677                     s->ptr_lastgob += current_packet_size;
2678                     s->first_slice_line=1;
2679                     s->resync_mb_x=mb_x;
2680                     s->resync_mb_y=mb_y;
2681                 }
2682             }
2683
2684             if(  (s->resync_mb_x   == s->mb_x)
2685                && s->resync_mb_y+1 == s->mb_y){
2686                 s->first_slice_line=0;
2687             }
2688
2689             s->mb_skipped=0;
2690             s->dquant=0; //only for QP_RD
2691
2692             update_mb_info(s, 0);
2693
2694             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
2695                 int next_block=0;
2696                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2697
2698                 copy_context_before_encode(&backup_s, s, -1);
2699                 backup_s.pb= s->pb;
2700                 best_s.data_partitioning= s->data_partitioning;
2701                 best_s.partitioned_frame= s->partitioned_frame;
2702                 if(s->data_partitioning){
2703                     backup_s.pb2= s->pb2;
2704                     backup_s.tex_pb= s->tex_pb;
2705                 }
2706
2707                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2708                     s->mv_dir = MV_DIR_FORWARD;
2709                     s->mv_type = MV_TYPE_16X16;
2710                     s->mb_intra= 0;
2711                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2712                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2713                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2714                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2715                 }
2716                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2717                     s->mv_dir = MV_DIR_FORWARD;
2718                     s->mv_type = MV_TYPE_FIELD;
2719                     s->mb_intra= 0;
2720                     for(i=0; i<2; i++){
2721                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2722                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2723                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2724                     }
2725                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2726                                  &dmin, &next_block, 0, 0);
2727                 }
2728                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2729                     s->mv_dir = MV_DIR_FORWARD;
2730                     s->mv_type = MV_TYPE_16X16;
2731                     s->mb_intra= 0;
2732                     s->mv[0][0][0] = 0;
2733                     s->mv[0][0][1] = 0;
2734                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2735                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2736                 }
2737                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2738                     s->mv_dir = MV_DIR_FORWARD;
2739                     s->mv_type = MV_TYPE_8X8;
2740                     s->mb_intra= 0;
2741                     for(i=0; i<4; i++){
2742                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2743                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2744                     }
2745                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2746                                  &dmin, &next_block, 0, 0);
2747                 }
2748                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2749                     s->mv_dir = MV_DIR_FORWARD;
2750                     s->mv_type = MV_TYPE_16X16;
2751                     s->mb_intra= 0;
2752                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2753                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2754                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2755                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2756                 }
2757                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2758                     s->mv_dir = MV_DIR_BACKWARD;
2759                     s->mv_type = MV_TYPE_16X16;
2760                     s->mb_intra= 0;
2761                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2762                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2763                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2764                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2765                 }
2766                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2767                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2768                     s->mv_type = MV_TYPE_16X16;
2769                     s->mb_intra= 0;
2770                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2771                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2772                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2773                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2774                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2775                                  &dmin, &next_block, 0, 0);
2776                 }
2777                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2778                     s->mv_dir = MV_DIR_FORWARD;
2779                     s->mv_type = MV_TYPE_FIELD;
2780                     s->mb_intra= 0;
2781                     for(i=0; i<2; i++){
2782                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2783                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2784                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2785                     }
2786                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2787                                  &dmin, &next_block, 0, 0);
2788                 }
2789                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2790                     s->mv_dir = MV_DIR_BACKWARD;
2791                     s->mv_type = MV_TYPE_FIELD;
2792                     s->mb_intra= 0;
2793                     for(i=0; i<2; i++){
2794                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2795                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2796                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2797                     }
2798                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2799                                  &dmin, &next_block, 0, 0);
2800                 }
2801                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2802                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2803                     s->mv_type = MV_TYPE_FIELD;
2804                     s->mb_intra= 0;
2805                     for(dir=0; dir<2; dir++){
2806                         for(i=0; i<2; i++){
2807                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2808                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2809                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2810                         }
2811                     }
2812                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2813                                  &dmin, &next_block, 0, 0);
2814                 }
2815                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2816                     s->mv_dir = 0;
2817                     s->mv_type = MV_TYPE_16X16;
2818                     s->mb_intra= 1;
2819                     s->mv[0][0][0] = 0;
2820                     s->mv[0][0][1] = 0;
2821                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2822                                  &dmin, &next_block, 0, 0);
2823                     if(s->h263_pred || s->h263_aic){
2824                         if(best_s.mb_intra)
2825                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2826                         else
2827                             ff_clean_intra_table_entries(s); //old mode?
2828                     }
2829                 }
2830
2831                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
2832                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2833                         const int last_qp= backup_s.qscale;
2834                         int qpi, qp, dc[6];
2835                         int16_t ac[6][16];
2836                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2837                         static const int dquant_tab[4]={-1,1,-2,2};
2838
2839                         assert(backup_s.dquant == 0);
2840
2841                         //FIXME intra
2842                         s->mv_dir= best_s.mv_dir;
2843                         s->mv_type = MV_TYPE_16X16;
2844                         s->mb_intra= best_s.mb_intra;
2845                         s->mv[0][0][0] = best_s.mv[0][0][0];
2846                         s->mv[0][0][1] = best_s.mv[0][0][1];
2847                         s->mv[1][0][0] = best_s.mv[1][0][0];
2848                         s->mv[1][0][1] = best_s.mv[1][0][1];
2849
2850                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
2851                         for(; qpi<4; qpi++){
2852                             int dquant= dquant_tab[qpi];
2853                             qp= last_qp + dquant;
2854                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2855                                 continue;
2856                             backup_s.dquant= dquant;
2857                             if(s->mb_intra && s->dc_val[0]){
2858                                 for(i=0; i<6; i++){
2859                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2860                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(int16_t)*16);
2861                                 }
2862                             }
2863
2864                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2865                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2866                             if(best_s.qscale != qp){
2867                                 if(s->mb_intra && s->dc_val[0]){
2868                                     for(i=0; i<6; i++){
2869                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2870                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(int16_t)*16);
2871                                     }
2872                                 }
2873                             }
2874                         }
2875                     }
2876                 }
2877                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2878                     int mx= s->b_direct_mv_table[xy][0];
2879                     int my= s->b_direct_mv_table[xy][1];
2880
2881                     backup_s.dquant = 0;
2882                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2883                     s->mb_intra= 0;
2884                     ff_mpeg4_set_direct_mv(s, mx, my);
2885                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2886                                  &dmin, &next_block, mx, my);
2887                 }
2888                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2889                     backup_s.dquant = 0;
2890                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2891                     s->mb_intra= 0;
2892                     ff_mpeg4_set_direct_mv(s, 0, 0);
2893                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2894                                  &dmin, &next_block, 0, 0);
2895                 }
2896                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
2897                     int coded=0;
2898                     for(i=0; i<6; i++)
2899                         coded |= s->block_last_index[i];
2900                     if(coded){
2901                         int mx,my;
2902                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2903                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
2904                             mx=my=0; //FIXME find the one we actually used
2905                             ff_mpeg4_set_direct_mv(s, mx, my);
2906                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2907                             mx= s->mv[1][0][0];
2908                             my= s->mv[1][0][1];
2909                         }else{
2910                             mx= s->mv[0][0][0];
2911                             my= s->mv[0][0][1];
2912                         }
2913
2914                         s->mv_dir= best_s.mv_dir;
2915                         s->mv_type = best_s.mv_type;
2916                         s->mb_intra= 0;
2917 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2918                         s->mv[0][0][1] = best_s.mv[0][0][1];
2919                         s->mv[1][0][0] = best_s.mv[1][0][0];
2920                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2921                         backup_s.dquant= 0;
2922                         s->skipdct=1;
2923                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2924                                         &dmin, &next_block, mx, my);
2925                         s->skipdct=0;
2926                     }
2927                 }
2928
2929                 s->current_picture.qscale_table[xy] = best_s.qscale;
2930
2931                 copy_context_after_encode(s, &best_s, -1);
2932
2933                 pb_bits_count= put_bits_count(&s->pb);
2934                 flush_put_bits(&s->pb);
2935                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2936                 s->pb= backup_s.pb;
2937
2938                 if(s->data_partitioning){
2939                     pb2_bits_count= put_bits_count(&s->pb2);
2940                     flush_put_bits(&s->pb2);
2941                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2942                     s->pb2= backup_s.pb2;
2943
2944                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2945                     flush_put_bits(&s->tex_pb);
2946                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2947                     s->tex_pb= backup_s.tex_pb;
2948                 }
2949                 s->last_bits= put_bits_count(&s->pb);
2950
2951                 if (CONFIG_H263_ENCODER &&
2952                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2953                     ff_h263_update_motion_val(s);
2954
2955                 if(next_block==0){ //FIXME 16 vs linesize16
2956                     s->hdsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
2957                     s->hdsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2958                     s->hdsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2959                 }
2960
2961                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
2962                     ff_MPV_decode_mb(s, s->block);
2963             } else {
2964                 int motion_x = 0, motion_y = 0;
2965                 s->mv_type=MV_TYPE_16X16;
2966                 // only one MB-Type possible
2967
2968                 switch(mb_type){
2969                 case CANDIDATE_MB_TYPE_INTRA:
2970                     s->mv_dir = 0;
2971                     s->mb_intra= 1;
2972                     motion_x= s->mv[0][0][0] = 0;
2973                     motion_y= s->mv[0][0][1] = 0;
2974                     break;
2975                 case CANDIDATE_MB_TYPE_INTER:
2976                     s->mv_dir = MV_DIR_FORWARD;
2977                     s->mb_intra= 0;
2978                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
2979                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
2980                     break;
2981                 case CANDIDATE_MB_TYPE_INTER_I:
2982                     s->mv_dir = MV_DIR_FORWARD;
2983                     s->mv_type = MV_TYPE_FIELD;
2984                     s->mb_intra= 0;
2985                     for(i=0; i<2; i++){
2986                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2987                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2988                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2989                     }
2990                     break;
2991                 case CANDIDATE_MB_TYPE_INTER4V:
2992                     s->mv_dir = MV_DIR_FORWARD;
2993                     s->mv_type = MV_TYPE_8X8;
2994                     s->mb_intra= 0;
2995                     for(i=0; i<4; i++){
2996                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2997                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2998                     }
2999                     break;
3000                 case CANDIDATE_MB_TYPE_DIRECT:
3001                     if (CONFIG_MPEG4_ENCODER) {
3002                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3003                         s->mb_intra= 0;
3004                         motion_x=s->b_direct_mv_table[xy][0];
3005                         motion_y=s->b_direct_mv_table[xy][1];
3006                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
3007                     }
3008                     break;
3009                 case CANDIDATE_MB_TYPE_DIRECT0:
3010                     if (CONFIG_MPEG4_ENCODER) {
3011                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3012                         s->mb_intra= 0;
3013                         ff_mpeg4_set_direct_mv(s, 0, 0);
3014                     }
3015                     break;
3016                 case CANDIDATE_MB_TYPE_BIDIR:
3017                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3018                     s->mb_intra= 0;
3019                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3020                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3021                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3022                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3023                     break;
3024                 case CANDIDATE_MB_TYPE_BACKWARD:
3025                     s->mv_dir = MV_DIR_BACKWARD;
3026                     s->mb_intra= 0;
3027                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3028                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3029                     break;
3030                 case CANDIDATE_MB_TYPE_FORWARD:
3031                     s->mv_dir = MV_DIR_FORWARD;
3032                     s->mb_intra= 0;
3033                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3034                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3035                     break;
3036                 case CANDIDATE_MB_TYPE_FORWARD_I:
3037                     s->mv_dir = MV_DIR_FORWARD;
3038                     s->mv_type = MV_TYPE_FIELD;
3039                     s->mb_intra= 0;
3040                     for(i=0; i<2; i++){
3041                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3042                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3043                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3044                     }
3045                     break;
3046                 case CANDIDATE_MB_TYPE_BACKWARD_I:
3047                     s->mv_dir = MV_DIR_BACKWARD;
3048                     s->mv_type = MV_TYPE_FIELD;
3049                     s->mb_intra= 0;
3050                     for(i=0; i<2; i++){
3051                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3052                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3053                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3054                     }
3055                     break;
3056                 case CANDIDATE_MB_TYPE_BIDIR_I:
3057                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3058                     s->mv_type = MV_TYPE_FIELD;
3059                     s->mb_intra= 0;
3060                     for(dir=0; dir<2; dir++){
3061                         for(i=0; i<2; i++){
3062                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3063                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3064                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3065                         }
3066                     }
3067                     break;
3068                 default:
3069                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
3070                 }
3071
3072                 encode_mb(s, motion_x, motion_y);
3073
3074                 // RAL: Update last macroblock type
3075                 s->last_mv_dir = s->mv_dir;
3076
3077                 if (CONFIG_H263_ENCODER &&
3078                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3079                     ff_h263_update_motion_val(s);
3080
3081                 ff_MPV_decode_mb(s, s->block);
3082             }
3083
3084             /* clean the MV table in IPS frames for direct mode in B frames */
3085             if(s->mb_intra /* && I,P,S_TYPE */){
3086                 s->p_mv_table[xy][0]=0;
3087                 s->p_mv_table[xy][1]=0;
3088             }
3089
3090             if(s->flags&CODEC_FLAG_PSNR){
3091                 int w= 16;
3092                 int h= 16;
3093
3094                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
3095                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
3096
3097                 s->current_picture.f->error[0] += sse(
3098                     s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
3099                     s->dest[0], w, h, s->linesize);
3100                 s->current_picture.f->error[1] += sse(
3101                     s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3102                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3103                 s->current_picture.f->error[2] += sse(
3104                     s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3105                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3106             }
3107             if(s->loop_filter){
3108                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
3109                     ff_h263_loop_filter(s);
3110             }
3111             av_dlog(s->avctx, "MB %d %d bits\n",
3112                     s->mb_x + s->mb_y * s->mb_stride, put_bits_count(&s->pb));
3113         }
3114     }
3115
3116     //not beautiful here but we must write it before flushing so it has to be here
3117     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
3118         ff_msmpeg4_encode_ext_header(s);
3119
3120     write_slice_end(s);
3121
3122     /* Send the last GOB if RTP */
3123     if (s->avctx->rtp_callback) {
3124         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
3125         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
3126         /* Call the RTP callback to send the last GOB */
3127         emms_c();
3128         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
3129     }
3130
3131     return 0;
3132 }
3133
3134 #define MERGE(field) dst->field += src->field; src->field=0
3135 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
3136     MERGE(me.scene_change_score);
3137     MERGE(me.mc_mb_var_sum_temp);
3138     MERGE(me.mb_var_sum_temp);
3139 }
3140
3141 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
3142     int i;
3143
3144     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
3145     MERGE(dct_count[1]);
3146     MERGE(mv_bits);
3147     MERGE(i_tex_bits);
3148     MERGE(p_tex_bits);
3149     MERGE(i_count);
3150     MERGE(f_count);
3151     MERGE(b_count);
3152     MERGE(skip_count);
3153     MERGE(misc_bits);
3154     MERGE(er.error_count);
3155     MERGE(padding_bug_score);
3156     MERGE(current_picture.f->error[0]);
3157     MERGE(current_picture.f->error[1]);
3158     MERGE(current_picture.f->error[2]);
3159
3160     if(dst->avctx->noise_reduction){
3161         for(i=0; i<64; i++){
3162             MERGE(dct_error_sum[0][i]);
3163             MERGE(dct_error_sum[1][i]);
3164         }
3165     }
3166
3167     assert(put_bits_count(&src->pb) % 8 ==0);
3168     assert(put_bits_count(&dst->pb) % 8 ==0);
3169     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3170     flush_put_bits(&dst->pb);
3171 }
3172
3173 static int estimate_qp(MpegEncContext *s, int dry_run){
3174     if (s->next_lambda){
3175         s->current_picture_ptr->f->quality =
3176         s->current_picture.f->quality = s->next_lambda;
3177         if(!dry_run) s->next_lambda= 0;
3178     } else if (!s->fixed_qscale) {
3179         s->current_picture_ptr->f->quality =
3180         s->current_picture.f->quality = ff_rate_estimate_qscale(s, dry_run);
3181         if (s->current_picture.f->quality < 0)
3182             return -1;
3183     }
3184
3185     if(s->adaptive_quant){
3186         switch(s->codec_id){
3187         case AV_CODEC_ID_MPEG4:
3188             if (CONFIG_MPEG4_ENCODER)
3189                 ff_clean_mpeg4_qscales(s);
3190             break;
3191         case AV_CODEC_ID_H263:
3192         case AV_CODEC_ID_H263P:
3193         case AV_CODEC_ID_FLV1:
3194             if (CONFIG_H263_ENCODER)
3195                 ff_clean_h263_qscales(s);
3196             break;
3197         default:
3198             ff_init_qscale_tab(s);
3199         }
3200
3201         s->lambda= s->lambda_table[0];
3202         //FIXME broken
3203     }else
3204         s->lambda = s->current_picture.f->quality;
3205     update_qscale(s);
3206     return 0;
3207 }
3208
3209 /* must be called before writing the header */
3210 static void set_frame_distances(MpegEncContext * s){
3211     assert(s->current_picture_ptr->f->pts != AV_NOPTS_VALUE);
3212     s->time = s->current_picture_ptr->f->pts * s->avctx->time_base.num;
3213
3214     if(s->pict_type==AV_PICTURE_TYPE_B){
3215         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3216         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3217     }else{
3218         s->pp_time= s->time - s->last_non_b_time;
3219         s->last_non_b_time= s->time;
3220         assert(s->picture_number==0 || s->pp_time > 0);
3221     }
3222 }
3223
3224 static int encode_picture(MpegEncContext *s, int picture_number)
3225 {
3226     int i, ret;
3227     int bits;
3228     int context_count = s->slice_context_count;
3229
3230     s->picture_number = picture_number;
3231
3232     /* Reset the average MB variance */
3233     s->me.mb_var_sum_temp    =
3234     s->me.mc_mb_var_sum_temp = 0;
3235
3236     /* we need to initialize some time vars before we can encode b-frames */
3237     // RAL: Condition added for MPEG1VIDEO
3238     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3239         set_frame_distances(s);
3240     if(CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4)
3241         ff_set_mpeg4_time(s);
3242
3243     s->me.scene_change_score=0;
3244
3245 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3246
3247     if(s->pict_type==AV_PICTURE_TYPE_I){
3248         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3249         else                        s->no_rounding=0;
3250     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3251         if(s->flipflop_rounding || s->codec_id == AV_CODEC_ID_H263P || s->codec_id == AV_CODEC_ID_MPEG4)
3252             s->no_rounding ^= 1;
3253     }
3254
3255     if(s->flags & CODEC_FLAG_PASS2){
3256         if (estimate_qp(s,1) < 0)
3257             return -1;
3258         ff_get_2pass_fcode(s);
3259     }else if(!(s->flags & CODEC_FLAG_QSCALE)){
3260         if(s->pict_type==AV_PICTURE_TYPE_B)
3261             s->lambda= s->last_lambda_for[s->pict_type];
3262         else
3263             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3264         update_qscale(s);
3265     }
3266
3267     s->mb_intra=0; //for the rate distortion & bit compare functions
3268     for(i=1; i<context_count; i++){
3269         ret = ff_update_duplicate_context(s->thread_context[i], s);
3270         if (ret < 0)
3271             return ret;
3272     }
3273
3274     if(ff_init_me(s)<0)
3275         return -1;
3276
3277     /* Estimate motion for every MB */
3278     if(s->pict_type != AV_PICTURE_TYPE_I){
3279         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
3280         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
3281         if (s->pict_type != AV_PICTURE_TYPE_B) {
3282             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
3283                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3284             }
3285         }
3286
3287         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3288     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3289         /* I-Frame */
3290         for(i=0; i<s->mb_stride*s->mb_height; i++)
3291             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3292
3293         if(!s->fixed_qscale){
3294             /* finding spatial complexity for I-frame rate control */
3295             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3296         }
3297     }
3298     for(i=1; i<context_count; i++){
3299         merge_context_after_me(s, s->thread_context[i]);
3300     }
3301     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3302     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3303     emms_c();
3304
3305     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
3306         s->pict_type= AV_PICTURE_TYPE_I;
3307         for(i=0; i<s->mb_stride*s->mb_height; i++)