34e008f66d9a8cb1cdfca2bff9d9d54d03ceaffd
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of Libav.
9  *
10  * Libav is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * Libav is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with Libav; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include <stdint.h>
31
32 #include "libavutil/internal.h"
33 #include "libavutil/intmath.h"
34 #include "libavutil/mathematics.h"
35 #include "libavutil/pixdesc.h"
36 #include "libavutil/opt.h"
37 #include "libavutil/timer.h"
38 #include "avcodec.h"
39 #include "dct.h"
40 #include "idctdsp.h"
41 #include "mpeg12.h"
42 #include "mpegvideo.h"
43 #include "mpegvideodata.h"
44 #include "h261.h"
45 #include "h263.h"
46 #include "h263data.h"
47 #include "mjpegenc_common.h"
48 #include "mathops.h"
49 #include "mpegutils.h"
50 #include "mjpegenc.h"
51 #include "msmpeg4.h"
52 #include "pixblockdsp.h"
53 #include "qpeldsp.h"
54 #include "faandct.h"
55 #include "thread.h"
56 #include "aandcttab.h"
57 #include "flv.h"
58 #include "mpeg4video.h"
59 #include "internal.h"
60 #include "bytestream.h"
61 #include "wmv2.h"
62 #include "rv10.h"
63 #include <limits.h>
64
65 #define QUANT_BIAS_SHIFT 8
66
67 #define QMAT_SHIFT_MMX 16
68 #define QMAT_SHIFT 22
69
70 static int encode_picture(MpegEncContext *s, int picture_number);
71 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
72 static int sse_mb(MpegEncContext *s);
73 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
74 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
75
76 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
77 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
78
79 const AVOption ff_mpv_generic_options[] = {
80     FF_MPV_COMMON_OPTS
81     { NULL },
82 };
83
84 void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64],
85                        uint16_t (*qmat16)[2][64],
86                        const uint16_t *quant_matrix,
87                        int bias, int qmin, int qmax, int intra)
88 {
89     FDCTDSPContext *fdsp = &s->fdsp;
90     int qscale;
91     int shift = 0;
92
93     for (qscale = qmin; qscale <= qmax; qscale++) {
94         int i;
95         if (fdsp->fdct == ff_jpeg_fdct_islow_8  ||
96 #if CONFIG_FAANDCT
97             fdsp->fdct == ff_faandct            ||
98 #endif /* CONFIG_FAANDCT */
99             fdsp->fdct == ff_jpeg_fdct_islow_10) {
100             for (i = 0; i < 64; i++) {
101                 const int j = s->idsp.idct_permutation[i];
102                 int64_t den = (int64_t) qscale * quant_matrix[j];
103                 /* 16 <= qscale * quant_matrix[i] <= 7905
104                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
105                  *             19952 <=              x  <= 249205026
106                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
107                  *           3444240 >= (1 << 36) / (x) >= 275 */
108
109                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
110             }
111         } else if (fdsp->fdct == ff_fdct_ifast) {
112             for (i = 0; i < 64; i++) {
113                 const int j = s->idsp.idct_permutation[i];
114                 int64_t den = ff_aanscales[i] * (int64_t) qscale * quant_matrix[j];
115                 /* 16 <= qscale * quant_matrix[i] <= 7905
116                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
117                  *             19952 <=              x  <= 249205026
118                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
119                  *           3444240 >= (1 << 36) / (x) >= 275 */
120
121                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / den);
122             }
123         } else {
124             for (i = 0; i < 64; i++) {
125                 const int j = s->idsp.idct_permutation[i];
126                 int64_t den = (int64_t) qscale * quant_matrix[j];
127                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
128                  * Assume x = qscale * quant_matrix[i]
129                  * So             16 <=              x  <= 7905
130                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
131                  * so          32768 >= (1 << 19) / (x) >= 67 */
132                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
133                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
134                 //                    (qscale * quant_matrix[i]);
135                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / den;
136
137                 if (qmat16[qscale][0][i] == 0 ||
138                     qmat16[qscale][0][i] == 128 * 256)
139                     qmat16[qscale][0][i] = 128 * 256 - 1;
140                 qmat16[qscale][1][i] =
141                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
142                                 qmat16[qscale][0][i]);
143             }
144         }
145
146         for (i = intra; i < 64; i++) {
147             int64_t max = 8191;
148             if (fdsp->fdct == ff_fdct_ifast) {
149                 max = (8191LL * ff_aanscales[i]) >> 14;
150             }
151             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
152                 shift++;
153             }
154         }
155     }
156     if (shift) {
157         av_log(NULL, AV_LOG_INFO,
158                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
159                QMAT_SHIFT - shift);
160     }
161 }
162
163 static inline void update_qscale(MpegEncContext *s)
164 {
165     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
166                 (FF_LAMBDA_SHIFT + 7);
167     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
168
169     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
170                  FF_LAMBDA_SHIFT;
171 }
172
173 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
174 {
175     int i;
176
177     if (matrix) {
178         put_bits(pb, 1, 1);
179         for (i = 0; i < 64; i++) {
180             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
181         }
182     } else
183         put_bits(pb, 1, 0);
184 }
185
186 /**
187  * init s->current_picture.qscale_table from s->lambda_table
188  */
189 void ff_init_qscale_tab(MpegEncContext *s)
190 {
191     int8_t * const qscale_table = s->current_picture.qscale_table;
192     int i;
193
194     for (i = 0; i < s->mb_num; i++) {
195         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
196         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
197         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
198                                                   s->avctx->qmax);
199     }
200 }
201
202 static void update_duplicate_context_after_me(MpegEncContext *dst,
203                                               MpegEncContext *src)
204 {
205 #define COPY(a) dst->a= src->a
206     COPY(pict_type);
207     COPY(current_picture);
208     COPY(f_code);
209     COPY(b_code);
210     COPY(qscale);
211     COPY(lambda);
212     COPY(lambda2);
213     COPY(picture_in_gop_number);
214     COPY(gop_picture_number);
215     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
216     COPY(progressive_frame);    // FIXME don't set in encode_header
217     COPY(partitioned_frame);    // FIXME don't set in encode_header
218 #undef COPY
219 }
220
221 /**
222  * Set the given MpegEncContext to defaults for encoding.
223  * the changed fields will not depend upon the prior state of the MpegEncContext.
224  */
225 static void mpv_encode_defaults(MpegEncContext *s)
226 {
227     int i;
228     ff_mpv_common_defaults(s);
229
230     for (i = -16; i < 16; i++) {
231         default_fcode_tab[i + MAX_MV] = 1;
232     }
233     s->me.mv_penalty = default_mv_penalty;
234     s->fcode_tab     = default_fcode_tab;
235
236     s->input_picture_number  = 0;
237     s->picture_in_gop_number = 0;
238 }
239
240 /* init video encoder */
241 av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
242 {
243     MpegEncContext *s = avctx->priv_data;
244     AVCPBProperties *cpb_props;
245     int i, ret, format_supported;
246
247     mpv_encode_defaults(s);
248
249     switch (avctx->codec_id) {
250     case AV_CODEC_ID_MPEG2VIDEO:
251         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
252             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
253             av_log(avctx, AV_LOG_ERROR,
254                    "only YUV420 and YUV422 are supported\n");
255             return -1;
256         }
257         break;
258     case AV_CODEC_ID_MJPEG:
259         format_supported = 0;
260         /* JPEG color space */
261         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
262             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
263             (avctx->color_range == AVCOL_RANGE_JPEG &&
264              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
265               avctx->pix_fmt == AV_PIX_FMT_YUV422P)))
266             format_supported = 1;
267         /* MPEG color space */
268         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
269                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
270                   avctx->pix_fmt == AV_PIX_FMT_YUV422P))
271             format_supported = 1;
272
273         if (!format_supported) {
274             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
275             return -1;
276         }
277         break;
278     default:
279         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
280             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
281             return -1;
282         }
283     }
284
285     switch (avctx->pix_fmt) {
286     case AV_PIX_FMT_YUVJ422P:
287     case AV_PIX_FMT_YUV422P:
288         s->chroma_format = CHROMA_422;
289         break;
290     case AV_PIX_FMT_YUVJ420P:
291     case AV_PIX_FMT_YUV420P:
292     default:
293         s->chroma_format = CHROMA_420;
294         break;
295     }
296
297     s->bit_rate = avctx->bit_rate;
298     s->width    = avctx->width;
299     s->height   = avctx->height;
300     if (avctx->gop_size > 600 &&
301         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
302         av_log(avctx, AV_LOG_ERROR,
303                "Warning keyframe interval too large! reducing it ...\n");
304         avctx->gop_size = 600;
305     }
306     s->gop_size     = avctx->gop_size;
307     s->avctx        = avctx;
308     if (avctx->max_b_frames > MAX_B_FRAMES) {
309         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
310                "is %d.\n", MAX_B_FRAMES);
311     }
312     s->max_b_frames = avctx->max_b_frames;
313     s->codec_id     = avctx->codec->id;
314     s->strict_std_compliance = avctx->strict_std_compliance;
315     s->quarter_sample     = (avctx->flags & AV_CODEC_FLAG_QPEL) != 0;
316     s->mpeg_quant         = avctx->mpeg_quant;
317     s->rtp_mode           = !!avctx->rtp_payload_size;
318     s->intra_dc_precision = avctx->intra_dc_precision;
319     s->user_specified_pts = AV_NOPTS_VALUE;
320
321     if (s->gop_size <= 1) {
322         s->intra_only = 1;
323         s->gop_size   = 12;
324     } else {
325         s->intra_only = 0;
326     }
327
328 #if FF_API_MOTION_EST
329 FF_DISABLE_DEPRECATION_WARNINGS
330     s->me_method = avctx->me_method;
331 FF_ENABLE_DEPRECATION_WARNINGS
332 #endif
333
334     /* Fixed QSCALE */
335     s->fixed_qscale = !!(avctx->flags & AV_CODEC_FLAG_QSCALE);
336
337 #if FF_API_MPV_OPT
338     FF_DISABLE_DEPRECATION_WARNINGS
339     if (avctx->border_masking != 0.0)
340         s->border_masking = avctx->border_masking;
341     FF_ENABLE_DEPRECATION_WARNINGS
342 #endif
343
344     s->adaptive_quant = (s->avctx->lumi_masking ||
345                          s->avctx->dark_masking ||
346                          s->avctx->temporal_cplx_masking ||
347                          s->avctx->spatial_cplx_masking  ||
348                          s->avctx->p_masking      ||
349                          s->border_masking ||
350                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
351                         !s->fixed_qscale;
352
353     s->loop_filter = !!(s->avctx->flags & AV_CODEC_FLAG_LOOP_FILTER);
354
355     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
356         av_log(avctx, AV_LOG_ERROR,
357                "a vbv buffer size is needed, "
358                "for encoding with a maximum bitrate\n");
359         return -1;
360     }
361
362     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
363         av_log(avctx, AV_LOG_INFO,
364                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
365     }
366
367     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
368         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
369         return -1;
370     }
371
372     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
373         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
374         return -1;
375     }
376
377     if (avctx->rc_max_rate &&
378         avctx->rc_max_rate == avctx->bit_rate &&
379         avctx->rc_max_rate != avctx->rc_min_rate) {
380         av_log(avctx, AV_LOG_INFO,
381                "impossible bitrate constraints, this will fail\n");
382     }
383
384     if (avctx->rc_buffer_size &&
385         avctx->bit_rate * (int64_t)avctx->time_base.num >
386             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
387         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
388         return -1;
389     }
390
391     if (!s->fixed_qscale &&
392         avctx->bit_rate * av_q2d(avctx->time_base) >
393             avctx->bit_rate_tolerance) {
394         av_log(avctx, AV_LOG_ERROR,
395                "bitrate tolerance too small for bitrate\n");
396         return -1;
397     }
398
399     if (s->avctx->rc_max_rate &&
400         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
401         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
402          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
403         90000LL * (avctx->rc_buffer_size - 1) >
404             s->avctx->rc_max_rate * 0xFFFFLL) {
405         av_log(avctx, AV_LOG_INFO,
406                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
407                "specified vbv buffer is too large for the given bitrate!\n");
408     }
409
410     if ((s->avctx->flags & AV_CODEC_FLAG_4MV) && s->codec_id != AV_CODEC_ID_MPEG4 &&
411         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
412         s->codec_id != AV_CODEC_ID_FLV1) {
413         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
414         return -1;
415     }
416
417     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
418         av_log(avctx, AV_LOG_ERROR,
419                "OBMC is only supported with simple mb decision\n");
420         return -1;
421     }
422
423     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
424         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
425         return -1;
426     }
427
428     if (s->max_b_frames                    &&
429         s->codec_id != AV_CODEC_ID_MPEG4      &&
430         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
431         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
432         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
433         return -1;
434     }
435
436     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
437          s->codec_id == AV_CODEC_ID_H263  ||
438          s->codec_id == AV_CODEC_ID_H263P) &&
439         (avctx->sample_aspect_ratio.num > 255 ||
440          avctx->sample_aspect_ratio.den > 255)) {
441         av_log(avctx, AV_LOG_ERROR,
442                "Invalid pixel aspect ratio %i/%i, limit is 255/255\n",
443                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
444         return -1;
445     }
446
447     if ((s->avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT | AV_CODEC_FLAG_INTERLACED_ME)) &&
448         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
449         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
450         return -1;
451     }
452
453     // FIXME mpeg2 uses that too
454     if (s->mpeg_quant && s->codec_id != AV_CODEC_ID_MPEG4) {
455         av_log(avctx, AV_LOG_ERROR,
456                "mpeg2 style quantization not supported by codec\n");
457         return -1;
458     }
459
460     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
461         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
462         return -1;
463     }
464
465     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
466         s->avctx->mb_decision != FF_MB_DECISION_RD) {
467         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
468         return -1;
469     }
470
471     if (s->avctx->scenechange_threshold < 1000000000 &&
472         (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)) {
473         av_log(avctx, AV_LOG_ERROR,
474                "closed gop with scene change detection are not supported yet, "
475                "set threshold to 1000000000\n");
476         return -1;
477     }
478
479     if (s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY) {
480         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
481             av_log(avctx, AV_LOG_ERROR,
482                   "low delay forcing is only available for mpeg2\n");
483             return -1;
484         }
485         if (s->max_b_frames != 0) {
486             av_log(avctx, AV_LOG_ERROR,
487                    "b frames cannot be used with low delay\n");
488             return -1;
489         }
490     }
491
492     if (s->q_scale_type == 1) {
493         if (avctx->qmax > 12) {
494             av_log(avctx, AV_LOG_ERROR,
495                    "non linear quant only supports qmax <= 12 currently\n");
496             return -1;
497         }
498     }
499
500     if (avctx->slices > 1 &&
501         (avctx->codec_id == AV_CODEC_ID_FLV1 || avctx->codec_id == AV_CODEC_ID_H261)) {
502         av_log(avctx, AV_LOG_ERROR, "Multiple slices are not supported by this codec\n");
503         return AVERROR(EINVAL);
504     }
505
506     if (s->avctx->thread_count > 1         &&
507         s->codec_id != AV_CODEC_ID_MPEG4      &&
508         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
509         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
510         (s->codec_id != AV_CODEC_ID_H263P)) {
511         av_log(avctx, AV_LOG_ERROR,
512                "multi threaded encoding not supported by codec\n");
513         return -1;
514     }
515
516     if (s->avctx->thread_count < 1) {
517         av_log(avctx, AV_LOG_ERROR,
518                "automatic thread number detection not supported by codec,"
519                "patch welcome\n");
520         return -1;
521     }
522
523     if (!avctx->time_base.den || !avctx->time_base.num) {
524         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
525         return -1;
526     }
527
528     if (avctx->b_frame_strategy && (avctx->flags & AV_CODEC_FLAG_PASS2)) {
529         av_log(avctx, AV_LOG_INFO,
530                "notice: b_frame_strategy only affects the first pass\n");
531         avctx->b_frame_strategy = 0;
532     }
533
534     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
535     if (i > 1) {
536         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
537         avctx->time_base.den /= i;
538         avctx->time_base.num /= i;
539         //return -1;
540     }
541
542     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
543         s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG) {
544         // (a + x * 3 / 8) / x
545         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
546         s->inter_quant_bias = 0;
547     } else {
548         s->intra_quant_bias = 0;
549         // (a - x / 4) / x
550         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
551     }
552
553 #if FF_API_QUANT_BIAS
554 FF_DISABLE_DEPRECATION_WARNINGS
555     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
556         s->intra_quant_bias = avctx->intra_quant_bias;
557     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
558         s->inter_quant_bias = avctx->inter_quant_bias;
559 FF_ENABLE_DEPRECATION_WARNINGS
560 #endif
561
562     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
563         s->avctx->time_base.den > (1 << 16) - 1) {
564         av_log(avctx, AV_LOG_ERROR,
565                "timebase %d/%d not supported by MPEG 4 standard, "
566                "the maximum admitted value for the timebase denominator "
567                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
568                (1 << 16) - 1);
569         return -1;
570     }
571     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
572
573     switch (avctx->codec->id) {
574     case AV_CODEC_ID_MPEG1VIDEO:
575         s->out_format = FMT_MPEG1;
576         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
577         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
578         break;
579     case AV_CODEC_ID_MPEG2VIDEO:
580         s->out_format = FMT_MPEG1;
581         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
582         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
583         s->rtp_mode   = 1;
584         break;
585     case AV_CODEC_ID_MJPEG:
586         s->out_format = FMT_MJPEG;
587         s->intra_only = 1; /* force intra only for jpeg */
588         if (!CONFIG_MJPEG_ENCODER ||
589             ff_mjpeg_encode_init(s) < 0)
590             return -1;
591         avctx->delay = 0;
592         s->low_delay = 1;
593         break;
594     case AV_CODEC_ID_H261:
595         if (!CONFIG_H261_ENCODER)
596             return -1;
597         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
598             av_log(avctx, AV_LOG_ERROR,
599                    "The specified picture size of %dx%d is not valid for the "
600                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
601                     s->width, s->height);
602             return -1;
603         }
604         s->out_format = FMT_H261;
605         avctx->delay  = 0;
606         s->low_delay  = 1;
607         s->rtp_mode   = 0; /* Sliced encoding not supported */
608         break;
609     case AV_CODEC_ID_H263:
610         if (!CONFIG_H263_ENCODER)
611         return -1;
612         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
613                              s->width, s->height) == 8) {
614             av_log(avctx, AV_LOG_INFO,
615                    "The specified picture size of %dx%d is not valid for "
616                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
617                    "352x288, 704x576, and 1408x1152."
618                    "Try H.263+.\n", s->width, s->height);
619             return -1;
620         }
621         s->out_format = FMT_H263;
622         avctx->delay  = 0;
623         s->low_delay  = 1;
624         break;
625     case AV_CODEC_ID_H263P:
626         s->out_format = FMT_H263;
627         s->h263_plus  = 1;
628         /* Fx */
629         s->h263_aic        = (avctx->flags & AV_CODEC_FLAG_AC_PRED) ? 1 : 0;
630         s->modified_quant  = s->h263_aic;
631         s->loop_filter     = (avctx->flags & AV_CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
632         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
633
634         /* /Fx */
635         /* These are just to be sure */
636         avctx->delay = 0;
637         s->low_delay = 1;
638         break;
639     case AV_CODEC_ID_FLV1:
640         s->out_format      = FMT_H263;
641         s->h263_flv        = 2; /* format = 1; 11-bit codes */
642         s->unrestricted_mv = 1;
643         s->rtp_mode  = 0; /* don't allow GOB */
644         avctx->delay = 0;
645         s->low_delay = 1;
646         break;
647     case AV_CODEC_ID_RV10:
648         s->out_format = FMT_H263;
649         avctx->delay  = 0;
650         s->low_delay  = 1;
651         break;
652     case AV_CODEC_ID_RV20:
653         s->out_format      = FMT_H263;
654         avctx->delay       = 0;
655         s->low_delay       = 1;
656         s->modified_quant  = 1;
657         s->h263_aic        = 1;
658         s->h263_plus       = 1;
659         s->loop_filter     = 1;
660         s->unrestricted_mv = 0;
661         break;
662     case AV_CODEC_ID_MPEG4:
663         s->out_format      = FMT_H263;
664         s->h263_pred       = 1;
665         s->unrestricted_mv = 1;
666         s->low_delay       = s->max_b_frames ? 0 : 1;
667         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
668         break;
669     case AV_CODEC_ID_MSMPEG4V2:
670         s->out_format      = FMT_H263;
671         s->h263_pred       = 1;
672         s->unrestricted_mv = 1;
673         s->msmpeg4_version = 2;
674         avctx->delay       = 0;
675         s->low_delay       = 1;
676         break;
677     case AV_CODEC_ID_MSMPEG4V3:
678         s->out_format        = FMT_H263;
679         s->h263_pred         = 1;
680         s->unrestricted_mv   = 1;
681         s->msmpeg4_version   = 3;
682         s->flipflop_rounding = 1;
683         avctx->delay         = 0;
684         s->low_delay         = 1;
685         break;
686     case AV_CODEC_ID_WMV1:
687         s->out_format        = FMT_H263;
688         s->h263_pred         = 1;
689         s->unrestricted_mv   = 1;
690         s->msmpeg4_version   = 4;
691         s->flipflop_rounding = 1;
692         avctx->delay         = 0;
693         s->low_delay         = 1;
694         break;
695     case AV_CODEC_ID_WMV2:
696         s->out_format        = FMT_H263;
697         s->h263_pred         = 1;
698         s->unrestricted_mv   = 1;
699         s->msmpeg4_version   = 5;
700         s->flipflop_rounding = 1;
701         avctx->delay         = 0;
702         s->low_delay         = 1;
703         break;
704     default:
705         return -1;
706     }
707
708     avctx->has_b_frames = !s->low_delay;
709
710     s->encoding = 1;
711
712     s->progressive_frame    =
713     s->progressive_sequence = !(avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT |
714                                                 AV_CODEC_FLAG_INTERLACED_ME) ||
715                                 s->alternate_scan);
716
717     /* init */
718     ff_mpv_idct_init(s);
719     if (ff_mpv_common_init(s) < 0)
720         return -1;
721
722     if (ARCH_X86)
723         ff_mpv_encode_init_x86(s);
724
725     ff_fdctdsp_init(&s->fdsp, avctx);
726     ff_me_cmp_init(&s->mecc, avctx);
727     ff_mpegvideoencdsp_init(&s->mpvencdsp, avctx);
728     ff_pixblockdsp_init(&s->pdsp, avctx);
729     ff_qpeldsp_init(&s->qdsp);
730
731     if (s->msmpeg4_version) {
732         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
733                           2 * 2 * (MAX_LEVEL + 1) *
734                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
735     }
736     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
737
738     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
739     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
740     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
741     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
742     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
743                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
744     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
745                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
746
747     if (s->avctx->noise_reduction) {
748         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
749                           2 * 64 * sizeof(uint16_t), fail);
750     }
751
752     if (CONFIG_H263_ENCODER)
753         ff_h263dsp_init(&s->h263dsp);
754     if (!s->dct_quantize)
755         s->dct_quantize = ff_dct_quantize_c;
756     if (!s->denoise_dct)
757         s->denoise_dct  = denoise_dct_c;
758     s->fast_dct_quantize = s->dct_quantize;
759     if (avctx->trellis)
760         s->dct_quantize  = dct_quantize_trellis_c;
761
762     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
763         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
764
765     if (s->slice_context_count > 1) {
766         s->rtp_mode = 1;
767
768         if (avctx->codec_id == AV_CODEC_ID_H263 || avctx->codec_id == AV_CODEC_ID_H263P)
769             s->h263_slice_structured = 1;
770     }
771
772     s->quant_precision = 5;
773
774     ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      s->avctx->ildct_cmp);
775     ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->avctx->frame_skip_cmp);
776
777     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
778         ff_h261_encode_init(s);
779     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
780         ff_h263_encode_init(s);
781     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
782         if ((ret = ff_msmpeg4_encode_init(s)) < 0)
783             return ret;
784     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
785         && s->out_format == FMT_MPEG1)
786         ff_mpeg1_encode_init(s);
787
788     /* init q matrix */
789     for (i = 0; i < 64; i++) {
790         int j = s->idsp.idct_permutation[i];
791         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
792             s->mpeg_quant) {
793             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
794             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
795         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
796             s->intra_matrix[j] =
797             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
798         } else {
799             /* mpeg1/2 */
800             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
801             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
802         }
803         if (s->avctx->intra_matrix)
804             s->intra_matrix[j] = s->avctx->intra_matrix[i];
805         if (s->avctx->inter_matrix)
806             s->inter_matrix[j] = s->avctx->inter_matrix[i];
807     }
808
809     /* precompute matrix */
810     /* for mjpeg, we do include qscale in the matrix */
811     if (s->out_format != FMT_MJPEG) {
812         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
813                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
814                           31, 1);
815         ff_convert_matrix(s, s->q_inter_matrix, s->q_inter_matrix16,
816                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
817                           31, 0);
818     }
819
820     if (ff_rate_control_init(s) < 0)
821         return -1;
822
823 #if FF_API_ERROR_RATE
824     FF_DISABLE_DEPRECATION_WARNINGS
825     if (avctx->error_rate)
826         s->error_rate = avctx->error_rate;
827     FF_ENABLE_DEPRECATION_WARNINGS;
828 #endif
829
830 #if FF_API_NORMALIZE_AQP
831     FF_DISABLE_DEPRECATION_WARNINGS
832     if (avctx->flags & CODEC_FLAG_NORMALIZE_AQP)
833         s->mpv_flags |= FF_MPV_FLAG_NAQ;
834     FF_ENABLE_DEPRECATION_WARNINGS;
835 #endif
836
837 #if FF_API_MV0
838     FF_DISABLE_DEPRECATION_WARNINGS
839     if (avctx->flags & CODEC_FLAG_MV0)
840         s->mpv_flags |= FF_MPV_FLAG_MV0;
841     FF_ENABLE_DEPRECATION_WARNINGS
842 #endif
843
844 #if FF_API_MPV_OPT
845     FF_DISABLE_DEPRECATION_WARNINGS
846     if (avctx->rc_qsquish != 0.0)
847         s->rc_qsquish = avctx->rc_qsquish;
848     if (avctx->rc_qmod_amp != 0.0)
849         s->rc_qmod_amp = avctx->rc_qmod_amp;
850     if (avctx->rc_qmod_freq)
851         s->rc_qmod_freq = avctx->rc_qmod_freq;
852     if (avctx->rc_buffer_aggressivity != 1.0)
853         s->rc_buffer_aggressivity = avctx->rc_buffer_aggressivity;
854     if (avctx->rc_initial_cplx != 0.0)
855         s->rc_initial_cplx = avctx->rc_initial_cplx;
856     if (avctx->lmin)
857         s->lmin = avctx->lmin;
858     if (avctx->lmax)
859         s->lmax = avctx->lmax;
860
861     if (avctx->rc_eq) {
862         av_freep(&s->rc_eq);
863         s->rc_eq = av_strdup(avctx->rc_eq);
864         if (!s->rc_eq)
865             return AVERROR(ENOMEM);
866     }
867     FF_ENABLE_DEPRECATION_WARNINGS
868 #endif
869
870     if (avctx->b_frame_strategy == 2) {
871         for (i = 0; i < s->max_b_frames + 2; i++) {
872             s->tmp_frames[i] = av_frame_alloc();
873             if (!s->tmp_frames[i])
874                 return AVERROR(ENOMEM);
875
876             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
877             s->tmp_frames[i]->width  = s->width  >> avctx->brd_scale;
878             s->tmp_frames[i]->height = s->height >> avctx->brd_scale;
879
880             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
881             if (ret < 0)
882                 return ret;
883         }
884     }
885
886     cpb_props = ff_add_cpb_side_data(avctx);
887     if (!cpb_props)
888         return AVERROR(ENOMEM);
889     cpb_props->max_bitrate = avctx->rc_max_rate;
890     cpb_props->min_bitrate = avctx->rc_min_rate;
891     cpb_props->avg_bitrate = avctx->bit_rate;
892     cpb_props->buffer_size = avctx->rc_buffer_size;
893
894     return 0;
895 fail:
896     ff_mpv_encode_end(avctx);
897     return AVERROR_UNKNOWN;
898 }
899
900 av_cold int ff_mpv_encode_end(AVCodecContext *avctx)
901 {
902     MpegEncContext *s = avctx->priv_data;
903     int i;
904
905     ff_rate_control_uninit(s);
906
907     ff_mpv_common_end(s);
908     if (CONFIG_MJPEG_ENCODER &&
909         s->out_format == FMT_MJPEG)
910         ff_mjpeg_encode_close(s);
911
912     av_freep(&avctx->extradata);
913
914     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
915         av_frame_free(&s->tmp_frames[i]);
916
917     ff_free_picture_tables(&s->new_picture);
918     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
919
920     av_freep(&s->avctx->stats_out);
921     av_freep(&s->ac_stats);
922
923     av_freep(&s->q_intra_matrix);
924     av_freep(&s->q_inter_matrix);
925     av_freep(&s->q_intra_matrix16);
926     av_freep(&s->q_inter_matrix16);
927     av_freep(&s->input_picture);
928     av_freep(&s->reordered_input_picture);
929     av_freep(&s->dct_offset);
930
931     return 0;
932 }
933
934 static int get_sae(uint8_t *src, int ref, int stride)
935 {
936     int x,y;
937     int acc = 0;
938
939     for (y = 0; y < 16; y++) {
940         for (x = 0; x < 16; x++) {
941             acc += FFABS(src[x + y * stride] - ref);
942         }
943     }
944
945     return acc;
946 }
947
948 static int get_intra_count(MpegEncContext *s, uint8_t *src,
949                            uint8_t *ref, int stride)
950 {
951     int x, y, w, h;
952     int acc = 0;
953
954     w = s->width  & ~15;
955     h = s->height & ~15;
956
957     for (y = 0; y < h; y += 16) {
958         for (x = 0; x < w; x += 16) {
959             int offset = x + y * stride;
960             int sad  = s->mecc.sad[0](NULL, src + offset, ref + offset,
961                                       stride, 16);
962             int mean = (s->mpvencdsp.pix_sum(src + offset, stride) + 128) >> 8;
963             int sae  = get_sae(src + offset, mean, stride);
964
965             acc += sae + 500 < sad;
966         }
967     }
968     return acc;
969 }
970
971 static int alloc_picture(MpegEncContext *s, Picture *pic, int shared)
972 {
973     return ff_alloc_picture(s->avctx, pic, &s->me, &s->sc, shared, 1,
974                             s->chroma_x_shift, s->chroma_y_shift, s->out_format,
975                             s->mb_stride, s->mb_height, s->b8_stride,
976                             &s->linesize, &s->uvlinesize);
977 }
978
979 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
980 {
981     Picture *pic = NULL;
982     int64_t pts;
983     int i, display_picture_number = 0, ret;
984     int encoding_delay = s->max_b_frames ? s->max_b_frames
985                                          : (s->low_delay ? 0 : 1);
986     int flush_offset = 1;
987     int direct = 1;
988
989     if (pic_arg) {
990         pts = pic_arg->pts;
991         display_picture_number = s->input_picture_number++;
992
993         if (pts != AV_NOPTS_VALUE) {
994             if (s->user_specified_pts != AV_NOPTS_VALUE) {
995                 int64_t time = pts;
996                 int64_t last = s->user_specified_pts;
997
998                 if (time <= last) {
999                     av_log(s->avctx, AV_LOG_ERROR,
1000                            "Error, Invalid timestamp=%"PRId64", "
1001                            "last=%"PRId64"\n", pts, s->user_specified_pts);
1002                     return -1;
1003                 }
1004
1005                 if (!s->low_delay && display_picture_number == 1)
1006                     s->dts_delta = time - last;
1007             }
1008             s->user_specified_pts = pts;
1009         } else {
1010             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1011                 s->user_specified_pts =
1012                 pts = s->user_specified_pts + 1;
1013                 av_log(s->avctx, AV_LOG_INFO,
1014                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
1015                        pts);
1016             } else {
1017                 pts = display_picture_number;
1018             }
1019         }
1020
1021         if (!pic_arg->buf[0] ||
1022             pic_arg->linesize[0] != s->linesize ||
1023             pic_arg->linesize[1] != s->uvlinesize ||
1024             pic_arg->linesize[2] != s->uvlinesize)
1025             direct = 0;
1026         if ((s->width & 15) || (s->height & 15))
1027             direct = 0;
1028
1029         ff_dlog(s->avctx, "%d %d %td %td\n", pic_arg->linesize[0],
1030                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
1031
1032         i = ff_find_unused_picture(s->avctx, s->picture, direct);
1033         if (i < 0)
1034             return i;
1035
1036         pic = &s->picture[i];
1037         pic->reference = 3;
1038
1039         if (direct) {
1040             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
1041                 return ret;
1042         }
1043         ret = alloc_picture(s, pic, direct);
1044         if (ret < 0)
1045             return ret;
1046
1047         if (!direct) {
1048             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1049                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1050                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1051                 // empty
1052             } else {
1053                 int h_chroma_shift, v_chroma_shift;
1054                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
1055                                                  &h_chroma_shift,
1056                                                  &v_chroma_shift);
1057
1058                 for (i = 0; i < 3; i++) {
1059                     int src_stride = pic_arg->linesize[i];
1060                     int dst_stride = i ? s->uvlinesize : s->linesize;
1061                     int h_shift = i ? h_chroma_shift : 0;
1062                     int v_shift = i ? v_chroma_shift : 0;
1063                     int w = s->width  >> h_shift;
1064                     int h = s->height >> v_shift;
1065                     uint8_t *src = pic_arg->data[i];
1066                     uint8_t *dst = pic->f->data[i];
1067
1068                     if (!s->avctx->rc_buffer_size)
1069                         dst += INPLACE_OFFSET;
1070
1071                     if (src_stride == dst_stride)
1072                         memcpy(dst, src, src_stride * h);
1073                     else {
1074                         int h2 = h;
1075                         uint8_t *dst2 = dst;
1076                         while (h2--) {
1077                             memcpy(dst2, src, w);
1078                             dst2 += dst_stride;
1079                             src += src_stride;
1080                         }
1081                     }
1082                     if ((s->width & 15) || (s->height & 15)) {
1083                         s->mpvencdsp.draw_edges(dst, dst_stride,
1084                                                 w, h,
1085                                                 16 >> h_shift,
1086                                                 16 >> v_shift,
1087                                                 EDGE_BOTTOM);
1088                     }
1089                 }
1090             }
1091         }
1092         ret = av_frame_copy_props(pic->f, pic_arg);
1093         if (ret < 0)
1094             return ret;
1095
1096         pic->f->display_picture_number = display_picture_number;
1097         pic->f->pts = pts; // we set this here to avoid modifiying pic_arg
1098     } else {
1099         /* Flushing: When we have not received enough input frames,
1100          * ensure s->input_picture[0] contains the first picture */
1101         for (flush_offset = 0; flush_offset < encoding_delay + 1; flush_offset++)
1102             if (s->input_picture[flush_offset])
1103                 break;
1104
1105         if (flush_offset <= 1)
1106             flush_offset = 1;
1107         else
1108             encoding_delay = encoding_delay - flush_offset + 1;
1109     }
1110
1111     /* shift buffer entries */
1112     for (i = flush_offset; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1113         s->input_picture[i - flush_offset] = s->input_picture[i];
1114
1115     s->input_picture[encoding_delay] = (Picture*) pic;
1116
1117     return 0;
1118 }
1119
1120 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1121 {
1122     int x, y, plane;
1123     int score = 0;
1124     int64_t score64 = 0;
1125
1126     for (plane = 0; plane < 3; plane++) {
1127         const int stride = p->f->linesize[plane];
1128         const int bw = plane ? 1 : 2;
1129         for (y = 0; y < s->mb_height * bw; y++) {
1130             for (x = 0; x < s->mb_width * bw; x++) {
1131                 int off = p->shared ? 0 : 16;
1132                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1133                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1134                 int v = s->mecc.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1135
1136                 switch (s->avctx->frame_skip_exp) {
1137                 case 0: score    =  FFMAX(score, v);          break;
1138                 case 1: score   += FFABS(v);                  break;
1139                 case 2: score   += v * v;                     break;
1140                 case 3: score64 += FFABS(v * v * (int64_t)v); break;
1141                 case 4: score64 += v * v * (int64_t)(v * v);  break;
1142                 }
1143             }
1144         }
1145     }
1146
1147     if (score)
1148         score64 = score;
1149
1150     if (score64 < s->avctx->frame_skip_threshold)
1151         return 1;
1152     if (score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda) >> 8))
1153         return 1;
1154     return 0;
1155 }
1156
1157 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1158 {
1159     AVPacket pkt = { 0 };
1160     int ret, got_output;
1161
1162     av_init_packet(&pkt);
1163     ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
1164     if (ret < 0)
1165         return ret;
1166
1167     ret = pkt.size;
1168     av_packet_unref(&pkt);
1169     return ret;
1170 }
1171
1172 static int estimate_best_b_count(MpegEncContext *s)
1173 {
1174     AVCodec *codec    = avcodec_find_encoder(s->avctx->codec_id);
1175     AVCodecContext *c = avcodec_alloc_context3(NULL);
1176     const int scale = s->avctx->brd_scale;
1177     int i, j, out_size, p_lambda, b_lambda, lambda2;
1178     int64_t best_rd  = INT64_MAX;
1179     int best_b_count = -1;
1180
1181     if (!c)
1182         return AVERROR(ENOMEM);
1183     assert(scale >= 0 && scale <= 3);
1184
1185     //emms_c();
1186     //s->next_picture_ptr->quality;
1187     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1188     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1189     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1190     if (!b_lambda) // FIXME we should do this somewhere else
1191         b_lambda = p_lambda;
1192     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1193                FF_LAMBDA_SHIFT;
1194
1195     c->width        = s->width  >> scale;
1196     c->height       = s->height >> scale;
1197     c->flags        = AV_CODEC_FLAG_QSCALE | AV_CODEC_FLAG_PSNR;
1198     c->flags       |= s->avctx->flags & AV_CODEC_FLAG_QPEL;
1199     c->mb_decision  = s->avctx->mb_decision;
1200     c->me_cmp       = s->avctx->me_cmp;
1201     c->mb_cmp       = s->avctx->mb_cmp;
1202     c->me_sub_cmp   = s->avctx->me_sub_cmp;
1203     c->pix_fmt      = AV_PIX_FMT_YUV420P;
1204     c->time_base    = s->avctx->time_base;
1205     c->max_b_frames = s->max_b_frames;
1206
1207     if (avcodec_open2(c, codec, NULL) < 0)
1208         return -1;
1209
1210     for (i = 0; i < s->max_b_frames + 2; i++) {
1211         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1212                                                 s->next_picture_ptr;
1213
1214         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1215             pre_input = *pre_input_ptr;
1216
1217             if (!pre_input.shared && i) {
1218                 pre_input.f->data[0] += INPLACE_OFFSET;
1219                 pre_input.f->data[1] += INPLACE_OFFSET;
1220                 pre_input.f->data[2] += INPLACE_OFFSET;
1221             }
1222
1223             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[0],
1224                                        s->tmp_frames[i]->linesize[0],
1225                                        pre_input.f->data[0],
1226                                        pre_input.f->linesize[0],
1227                                        c->width, c->height);
1228             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[1],
1229                                        s->tmp_frames[i]->linesize[1],
1230                                        pre_input.f->data[1],
1231                                        pre_input.f->linesize[1],
1232                                        c->width >> 1, c->height >> 1);
1233             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[2],
1234                                        s->tmp_frames[i]->linesize[2],
1235                                        pre_input.f->data[2],
1236                                        pre_input.f->linesize[2],
1237                                        c->width >> 1, c->height >> 1);
1238         }
1239     }
1240
1241     for (j = 0; j < s->max_b_frames + 1; j++) {
1242         int64_t rd = 0;
1243
1244         if (!s->input_picture[j])
1245             break;
1246
1247         c->error[0] = c->error[1] = c->error[2] = 0;
1248
1249         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1250         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1251
1252         out_size = encode_frame(c, s->tmp_frames[0]);
1253
1254         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1255
1256         for (i = 0; i < s->max_b_frames + 1; i++) {
1257             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1258
1259             s->tmp_frames[i + 1]->pict_type = is_p ?
1260                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1261             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1262
1263             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1264
1265             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1266         }
1267
1268         /* get the delayed frames */
1269         while (out_size) {
1270             out_size = encode_frame(c, NULL);
1271             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1272         }
1273
1274         rd += c->error[0] + c->error[1] + c->error[2];
1275
1276         if (rd < best_rd) {
1277             best_rd = rd;
1278             best_b_count = j;
1279         }
1280     }
1281
1282     avcodec_close(c);
1283     av_freep(&c);
1284
1285     return best_b_count;
1286 }
1287
1288 static int select_input_picture(MpegEncContext *s)
1289 {
1290     int i, ret;
1291
1292     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1293         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1294     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1295
1296     /* set next picture type & ordering */
1297     if (!s->reordered_input_picture[0] && s->input_picture[0]) {
1298         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1299             !s->next_picture_ptr || s->intra_only) {
1300             s->reordered_input_picture[0] = s->input_picture[0];
1301             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1302             s->reordered_input_picture[0]->f->coded_picture_number =
1303                 s->coded_picture_number++;
1304         } else {
1305             int b_frames;
1306
1307             if (s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor) {
1308                 if (s->picture_in_gop_number < s->gop_size &&
1309                     skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1310                     // FIXME check that te gop check above is +-1 correct
1311                     av_frame_unref(s->input_picture[0]->f);
1312
1313                     emms_c();
1314                     ff_vbv_update(s, 0);
1315
1316                     goto no_output_pic;
1317                 }
1318             }
1319
1320             if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
1321                 for (i = 0; i < s->max_b_frames + 1; i++) {
1322                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1323
1324                     if (pict_num >= s->rc_context.num_entries)
1325                         break;
1326                     if (!s->input_picture[i]) {
1327                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1328                         break;
1329                     }
1330
1331                     s->input_picture[i]->f->pict_type =
1332                         s->rc_context.entry[pict_num].new_pict_type;
1333                 }
1334             }
1335
1336             if (s->avctx->b_frame_strategy == 0) {
1337                 b_frames = s->max_b_frames;
1338                 while (b_frames && !s->input_picture[b_frames])
1339                     b_frames--;
1340             } else if (s->avctx->b_frame_strategy == 1) {
1341                 for (i = 1; i < s->max_b_frames + 1; i++) {
1342                     if (s->input_picture[i] &&
1343                         s->input_picture[i]->b_frame_score == 0) {
1344                         s->input_picture[i]->b_frame_score =
1345                             get_intra_count(s,
1346                                             s->input_picture[i    ]->f->data[0],
1347                                             s->input_picture[i - 1]->f->data[0],
1348                                             s->linesize) + 1;
1349                     }
1350                 }
1351                 for (i = 0; i < s->max_b_frames + 1; i++) {
1352                     if (!s->input_picture[i] ||
1353                         s->input_picture[i]->b_frame_score - 1 >
1354                             s->mb_num / s->avctx->b_sensitivity)
1355                         break;
1356                 }
1357
1358                 b_frames = FFMAX(0, i - 1);
1359
1360                 /* reset scores */
1361                 for (i = 0; i < b_frames + 1; i++) {
1362                     s->input_picture[i]->b_frame_score = 0;
1363                 }
1364             } else if (s->avctx->b_frame_strategy == 2) {
1365                 b_frames = estimate_best_b_count(s);
1366             } else {
1367                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1368                 b_frames = 0;
1369             }
1370
1371             emms_c();
1372
1373             for (i = b_frames - 1; i >= 0; i--) {
1374                 int type = s->input_picture[i]->f->pict_type;
1375                 if (type && type != AV_PICTURE_TYPE_B)
1376                     b_frames = i;
1377             }
1378             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1379                 b_frames == s->max_b_frames) {
1380                 av_log(s->avctx, AV_LOG_ERROR,
1381                        "warning, too many b frames in a row\n");
1382             }
1383
1384             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1385                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1386                     s->gop_size > s->picture_in_gop_number) {
1387                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1388                 } else {
1389                     if (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)
1390                         b_frames = 0;
1391                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1392                 }
1393             }
1394
1395             if ((s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP) && b_frames &&
1396                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1397                 b_frames--;
1398
1399             s->reordered_input_picture[0] = s->input_picture[b_frames];
1400             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1401                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1402             s->reordered_input_picture[0]->f->coded_picture_number =
1403                 s->coded_picture_number++;
1404             for (i = 0; i < b_frames; i++) {
1405                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1406                 s->reordered_input_picture[i + 1]->f->pict_type =
1407                     AV_PICTURE_TYPE_B;
1408                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1409                     s->coded_picture_number++;
1410             }
1411         }
1412     }
1413 no_output_pic:
1414     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1415
1416     if (s->reordered_input_picture[0]) {
1417         s->reordered_input_picture[0]->reference =
1418            s->reordered_input_picture[0]->f->pict_type !=
1419                AV_PICTURE_TYPE_B ? 3 : 0;
1420
1421         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->new_picture, s->reordered_input_picture[0])))
1422             return ret;
1423
1424         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1425             // input is a shared pix, so we can't modifiy it -> alloc a new
1426             // one & ensure that the shared one is reuseable
1427
1428             Picture *pic;
1429             int i = ff_find_unused_picture(s->avctx, s->picture, 0);
1430             if (i < 0)
1431                 return i;
1432             pic = &s->picture[i];
1433
1434             pic->reference = s->reordered_input_picture[0]->reference;
1435             if (alloc_picture(s, pic, 0) < 0) {
1436                 return -1;
1437             }
1438
1439             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1440             if (ret < 0)
1441                 return ret;
1442
1443             /* mark us unused / free shared pic */
1444             av_frame_unref(s->reordered_input_picture[0]->f);
1445             s->reordered_input_picture[0]->shared = 0;
1446
1447             s->current_picture_ptr = pic;
1448         } else {
1449             // input is not a shared pix -> reuse buffer for current_pix
1450             s->current_picture_ptr = s->reordered_input_picture[0];
1451             for (i = 0; i < 4; i++) {
1452                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1453             }
1454         }
1455         ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1456         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1457                                        s->current_picture_ptr)) < 0)
1458             return ret;
1459
1460         s->picture_number = s->new_picture.f->display_picture_number;
1461     }
1462     return 0;
1463 }
1464
1465 static void frame_end(MpegEncContext *s)
1466 {
1467     int i;
1468
1469     if (s->unrestricted_mv &&
1470         s->current_picture.reference &&
1471         !s->intra_only) {
1472         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1473         int hshift = desc->log2_chroma_w;
1474         int vshift = desc->log2_chroma_h;
1475         s->mpvencdsp.draw_edges(s->current_picture.f->data[0], s->linesize,
1476                                 s->h_edge_pos, s->v_edge_pos,
1477                                 EDGE_WIDTH, EDGE_WIDTH,
1478                                 EDGE_TOP | EDGE_BOTTOM);
1479         s->mpvencdsp.draw_edges(s->current_picture.f->data[1], s->uvlinesize,
1480                                 s->h_edge_pos >> hshift,
1481                                 s->v_edge_pos >> vshift,
1482                                 EDGE_WIDTH >> hshift,
1483                                 EDGE_WIDTH >> vshift,
1484                                 EDGE_TOP | EDGE_BOTTOM);
1485         s->mpvencdsp.draw_edges(s->current_picture.f->data[2], s->uvlinesize,
1486                                 s->h_edge_pos >> hshift,
1487                                 s->v_edge_pos >> vshift,
1488                                 EDGE_WIDTH >> hshift,
1489                                 EDGE_WIDTH >> vshift,
1490                                 EDGE_TOP | EDGE_BOTTOM);
1491     }
1492
1493     emms_c();
1494
1495     s->last_pict_type                 = s->pict_type;
1496     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1497     if (s->pict_type!= AV_PICTURE_TYPE_B)
1498         s->last_non_b_pict_type = s->pict_type;
1499
1500     if (s->encoding) {
1501         /* release non-reference frames */
1502         for (i = 0; i < MAX_PICTURE_COUNT; i++) {
1503             if (!s->picture[i].reference)
1504                 ff_mpeg_unref_picture(s->avctx, &s->picture[i]);
1505         }
1506     }
1507
1508 #if FF_API_CODED_FRAME
1509 FF_DISABLE_DEPRECATION_WARNINGS
1510     av_frame_copy_props(s->avctx->coded_frame, s->current_picture.f);
1511 FF_ENABLE_DEPRECATION_WARNINGS
1512 #endif
1513 #if FF_API_ERROR_FRAME
1514 FF_DISABLE_DEPRECATION_WARNINGS
1515     memcpy(s->current_picture.f->error, s->current_picture.encoding_error,
1516            sizeof(s->current_picture.encoding_error));
1517 FF_ENABLE_DEPRECATION_WARNINGS
1518 #endif
1519 }
1520
1521 static void update_noise_reduction(MpegEncContext *s)
1522 {
1523     int intra, i;
1524
1525     for (intra = 0; intra < 2; intra++) {
1526         if (s->dct_count[intra] > (1 << 16)) {
1527             for (i = 0; i < 64; i++) {
1528                 s->dct_error_sum[intra][i] >>= 1;
1529             }
1530             s->dct_count[intra] >>= 1;
1531         }
1532
1533         for (i = 0; i < 64; i++) {
1534             s->dct_offset[intra][i] = (s->avctx->noise_reduction *
1535                                        s->dct_count[intra] +
1536                                        s->dct_error_sum[intra][i] / 2) /
1537                                       (s->dct_error_sum[intra][i] + 1);
1538         }
1539     }
1540 }
1541
1542 static int frame_start(MpegEncContext *s)
1543 {
1544     int ret;
1545
1546     /* mark & release old frames */
1547     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1548         s->last_picture_ptr != s->next_picture_ptr &&
1549         s->last_picture_ptr->f->buf[0]) {
1550         ff_mpeg_unref_picture(s->avctx, s->last_picture_ptr);
1551     }
1552
1553     s->current_picture_ptr->f->pict_type = s->pict_type;
1554     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1555
1556     ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1557     if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1558                                    s->current_picture_ptr)) < 0)
1559         return ret;
1560
1561     if (s->pict_type != AV_PICTURE_TYPE_B) {
1562         s->last_picture_ptr = s->next_picture_ptr;
1563         if (!s->droppable)
1564             s->next_picture_ptr = s->current_picture_ptr;
1565     }
1566
1567     if (s->last_picture_ptr) {
1568         ff_mpeg_unref_picture(s->avctx, &s->last_picture);
1569         if (s->last_picture_ptr->f->buf[0] &&
1570             (ret = ff_mpeg_ref_picture(s->avctx, &s->last_picture,
1571                                        s->last_picture_ptr)) < 0)
1572             return ret;
1573     }
1574     if (s->next_picture_ptr) {
1575         ff_mpeg_unref_picture(s->avctx, &s->next_picture);
1576         if (s->next_picture_ptr->f->buf[0] &&
1577             (ret = ff_mpeg_ref_picture(s->avctx, &s->next_picture,
1578                                        s->next_picture_ptr)) < 0)
1579             return ret;
1580     }
1581
1582     if (s->picture_structure!= PICT_FRAME) {
1583         int i;
1584         for (i = 0; i < 4; i++) {
1585             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1586                 s->current_picture.f->data[i] +=
1587                     s->current_picture.f->linesize[i];
1588             }
1589             s->current_picture.f->linesize[i] *= 2;
1590             s->last_picture.f->linesize[i]    *= 2;
1591             s->next_picture.f->linesize[i]    *= 2;
1592         }
1593     }
1594
1595     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1596         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1597         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1598     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1599         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1600         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1601     } else {
1602         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1603         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1604     }
1605
1606     if (s->dct_error_sum) {
1607         assert(s->avctx->noise_reduction && s->encoding);
1608         update_noise_reduction(s);
1609     }
1610
1611     return 0;
1612 }
1613
1614 int ff_mpv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1615                           const AVFrame *pic_arg, int *got_packet)
1616 {
1617     MpegEncContext *s = avctx->priv_data;
1618     int i, stuffing_count, ret;
1619     int context_count = s->slice_context_count;
1620
1621     s->picture_in_gop_number++;
1622
1623     if (load_input_picture(s, pic_arg) < 0)
1624         return -1;
1625
1626     if (select_input_picture(s) < 0) {
1627         return -1;
1628     }
1629
1630     /* output? */
1631     if (s->new_picture.f->data[0]) {
1632         uint8_t *sd;
1633         if (!pkt->data &&
1634             (ret = ff_alloc_packet(pkt, s->mb_width*s->mb_height*MAX_MB_BYTES)) < 0)
1635             return ret;
1636         if (s->mb_info) {
1637             s->mb_info_ptr = av_packet_new_side_data(pkt,
1638                                  AV_PKT_DATA_H263_MB_INFO,
1639                                  s->mb_width*s->mb_height*12);
1640             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1641         }
1642
1643         for (i = 0; i < context_count; i++) {
1644             int start_y = s->thread_context[i]->start_mb_y;
1645             int   end_y = s->thread_context[i]->  end_mb_y;
1646             int h       = s->mb_height;
1647             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1648             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1649
1650             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1651         }
1652
1653         s->pict_type = s->new_picture.f->pict_type;
1654         //emms_c();
1655         ret = frame_start(s);
1656         if (ret < 0)
1657             return ret;
1658 vbv_retry:
1659         if (encode_picture(s, s->picture_number) < 0)
1660             return -1;
1661
1662 #if FF_API_STAT_BITS
1663 FF_DISABLE_DEPRECATION_WARNINGS
1664         avctx->header_bits = s->header_bits;
1665         avctx->mv_bits     = s->mv_bits;
1666         avctx->misc_bits   = s->misc_bits;
1667         avctx->i_tex_bits  = s->i_tex_bits;
1668         avctx->p_tex_bits  = s->p_tex_bits;
1669         avctx->i_count     = s->i_count;
1670         // FIXME f/b_count in avctx
1671         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1672         avctx->skip_count  = s->skip_count;
1673 FF_ENABLE_DEPRECATION_WARNINGS
1674 #endif
1675
1676         frame_end(s);
1677
1678         sd = av_packet_new_side_data(pkt, AV_PKT_DATA_QUALITY_FACTOR,
1679                                      sizeof(int));
1680         if (!sd)
1681             return AVERROR(ENOMEM);
1682         *(int *)sd = s->current_picture.f->quality;
1683
1684         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1685             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1686
1687         if (avctx->rc_buffer_size) {
1688             RateControlContext *rcc = &s->rc_context;
1689             int max_size = rcc->buffer_index * avctx->rc_max_available_vbv_use;
1690
1691             if (put_bits_count(&s->pb) > max_size &&
1692                 s->lambda < s->lmax) {
1693                 s->next_lambda = FFMAX(s->lambda + 1, s->lambda *
1694                                        (s->qscale + 1) / s->qscale);
1695                 if (s->adaptive_quant) {
1696                     int i;
1697                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1698                         s->lambda_table[i] =
1699                             FFMAX(s->lambda_table[i] + 1,
1700                                   s->lambda_table[i] * (s->qscale + 1) /
1701                                   s->qscale);
1702                 }
1703                 s->mb_skipped = 0;        // done in frame_start()
1704                 // done in encode_picture() so we must undo it
1705                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1706                     if (s->flipflop_rounding          ||
1707                         s->codec_id == AV_CODEC_ID_H263P ||
1708                         s->codec_id == AV_CODEC_ID_MPEG4)
1709                         s->no_rounding ^= 1;
1710                 }
1711                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1712                     s->time_base       = s->last_time_base;
1713                     s->last_non_b_time = s->time - s->pp_time;
1714                 }
1715                 for (i = 0; i < context_count; i++) {
1716                     PutBitContext *pb = &s->thread_context[i]->pb;
1717                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1718                 }
1719                 goto vbv_retry;
1720             }
1721
1722             assert(s->avctx->rc_max_rate);
1723         }
1724
1725         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
1726             ff_write_pass1_stats(s);
1727
1728         for (i = 0; i < 4; i++) {
1729             s->current_picture_ptr->encoding_error[i] = s->current_picture.encoding_error[i];
1730             avctx->error[i] += s->current_picture_ptr->encoding_error[i];
1731         }
1732
1733         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
1734             assert(put_bits_count(&s->pb) == s->header_bits + s->mv_bits +
1735                                              s->misc_bits + s->i_tex_bits +
1736                                              s->p_tex_bits);
1737         flush_put_bits(&s->pb);
1738         s->frame_bits  = put_bits_count(&s->pb);
1739
1740         stuffing_count = ff_vbv_update(s, s->frame_bits);
1741         if (stuffing_count) {
1742             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1743                     stuffing_count + 50) {
1744                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1745                 return -1;
1746             }
1747
1748             switch (s->codec_id) {
1749             case AV_CODEC_ID_MPEG1VIDEO:
1750             case AV_CODEC_ID_MPEG2VIDEO:
1751                 while (stuffing_count--) {
1752                     put_bits(&s->pb, 8, 0);
1753                 }
1754             break;
1755             case AV_CODEC_ID_MPEG4:
1756                 put_bits(&s->pb, 16, 0);
1757                 put_bits(&s->pb, 16, 0x1C3);
1758                 stuffing_count -= 4;
1759                 while (stuffing_count--) {
1760                     put_bits(&s->pb, 8, 0xFF);
1761                 }
1762             break;
1763             default:
1764                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1765             }
1766             flush_put_bits(&s->pb);
1767             s->frame_bits  = put_bits_count(&s->pb);
1768         }
1769
1770         /* update mpeg1/2 vbv_delay for CBR */
1771         if (s->avctx->rc_max_rate                          &&
1772             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1773             s->out_format == FMT_MPEG1                     &&
1774             90000LL * (avctx->rc_buffer_size - 1) <=
1775                 s->avctx->rc_max_rate * 0xFFFFLL) {
1776             AVCPBProperties *props;
1777             size_t props_size;
1778
1779             int vbv_delay, min_delay;
1780             double inbits  = s->avctx->rc_max_rate *
1781                              av_q2d(s->avctx->time_base);
1782             int    minbits = s->frame_bits - 8 *
1783                              (s->vbv_delay_ptr - s->pb.buf - 1);
1784             double bits    = s->rc_context.buffer_index + minbits - inbits;
1785
1786             if (bits < 0)
1787                 av_log(s->avctx, AV_LOG_ERROR,
1788                        "Internal error, negative bits\n");
1789
1790             assert(s->repeat_first_field == 0);
1791
1792             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
1793             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
1794                         s->avctx->rc_max_rate;
1795
1796             vbv_delay = FFMAX(vbv_delay, min_delay);
1797
1798             assert(vbv_delay < 0xFFFF);
1799
1800             s->vbv_delay_ptr[0] &= 0xF8;
1801             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
1802             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
1803             s->vbv_delay_ptr[2] &= 0x07;
1804             s->vbv_delay_ptr[2] |= vbv_delay << 3;
1805
1806             props = av_cpb_properties_alloc(&props_size);
1807             if (!props)
1808                 return AVERROR(ENOMEM);
1809             props->vbv_delay = vbv_delay * 300;
1810
1811             ret = av_packet_add_side_data(pkt, AV_PKT_DATA_CPB_PROPERTIES,
1812                                           (uint8_t*)props, props_size);
1813             if (ret < 0) {
1814                 av_freep(&props);
1815                 return ret;
1816             }
1817
1818 #if FF_API_VBV_DELAY
1819 FF_DISABLE_DEPRECATION_WARNINGS
1820             avctx->vbv_delay     = vbv_delay * 300;
1821 FF_ENABLE_DEPRECATION_WARNINGS
1822 #endif
1823         }
1824         s->total_bits     += s->frame_bits;
1825 #if FF_API_STAT_BITS
1826 FF_DISABLE_DEPRECATION_WARNINGS
1827         avctx->frame_bits  = s->frame_bits;
1828 FF_ENABLE_DEPRECATION_WARNINGS
1829 #endif
1830
1831
1832         pkt->pts = s->current_picture.f->pts;
1833         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
1834             if (!s->current_picture.f->coded_picture_number)
1835                 pkt->dts = pkt->pts - s->dts_delta;
1836             else
1837                 pkt->dts = s->reordered_pts;
1838             s->reordered_pts = pkt->pts;
1839         } else
1840             pkt->dts = pkt->pts;
1841         if (s->current_picture.f->key_frame)
1842             pkt->flags |= AV_PKT_FLAG_KEY;
1843         if (s->mb_info)
1844             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
1845     } else {
1846         s->frame_bits = 0;
1847     }
1848     assert((s->frame_bits & 7) == 0);
1849
1850     pkt->size = s->frame_bits / 8;
1851     *got_packet = !!pkt->size;
1852     return 0;
1853 }
1854
1855 static inline void dct_single_coeff_elimination(MpegEncContext *s,
1856                                                 int n, int threshold)
1857 {
1858     static const char tab[64] = {
1859         3, 2, 2, 1, 1, 1, 1, 1,
1860         1, 1, 1, 1, 1, 1, 1, 1,
1861         1, 1, 1, 1, 1, 1, 1, 1,
1862         0, 0, 0, 0, 0, 0, 0, 0,
1863         0, 0, 0, 0, 0, 0, 0, 0,
1864         0, 0, 0, 0, 0, 0, 0, 0,
1865         0, 0, 0, 0, 0, 0, 0, 0,
1866         0, 0, 0, 0, 0, 0, 0, 0
1867     };
1868     int score = 0;
1869     int run = 0;
1870     int i;
1871     int16_t *block = s->block[n];
1872     const int last_index = s->block_last_index[n];
1873     int skip_dc;
1874
1875     if (threshold < 0) {
1876         skip_dc = 0;
1877         threshold = -threshold;
1878     } else
1879         skip_dc = 1;
1880
1881     /* Are all we could set to zero already zero? */
1882     if (last_index <= skip_dc - 1)
1883         return;
1884
1885     for (i = 0; i <= last_index; i++) {
1886         const int j = s->intra_scantable.permutated[i];
1887         const int level = FFABS(block[j]);
1888         if (level == 1) {
1889             if (skip_dc && i == 0)
1890                 continue;
1891             score += tab[run];
1892             run = 0;
1893         } else if (level > 1) {
1894             return;
1895         } else {
1896             run++;
1897         }
1898     }
1899     if (score >= threshold)
1900         return;
1901     for (i = skip_dc; i <= last_index; i++) {
1902         const int j = s->intra_scantable.permutated[i];
1903         block[j] = 0;
1904     }
1905     if (block[0])
1906         s->block_last_index[n] = 0;
1907     else
1908         s->block_last_index[n] = -1;
1909 }
1910
1911 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
1912                                int last_index)
1913 {
1914     int i;
1915     const int maxlevel = s->max_qcoeff;
1916     const int minlevel = s->min_qcoeff;
1917     int overflow = 0;
1918
1919     if (s->mb_intra) {
1920         i = 1; // skip clipping of intra dc
1921     } else
1922         i = 0;
1923
1924     for (; i <= last_index; i++) {
1925         const int j = s->intra_scantable.permutated[i];
1926         int level = block[j];
1927
1928         if (level > maxlevel) {
1929             level = maxlevel;
1930             overflow++;
1931         } else if (level < minlevel) {
1932             level = minlevel;
1933             overflow++;
1934         }
1935
1936         block[j] = level;
1937     }
1938
1939     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1940         av_log(s->avctx, AV_LOG_INFO,
1941                "warning, clipping %d dct coefficients to %d..%d\n",
1942                overflow, minlevel, maxlevel);
1943 }
1944
1945 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
1946 {
1947     int x, y;
1948     // FIXME optimize
1949     for (y = 0; y < 8; y++) {
1950         for (x = 0; x < 8; x++) {
1951             int x2, y2;
1952             int sum = 0;
1953             int sqr = 0;
1954             int count = 0;
1955
1956             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
1957                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
1958                     int v = ptr[x2 + y2 * stride];
1959                     sum += v;
1960                     sqr += v * v;
1961                     count++;
1962                 }
1963             }
1964             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
1965         }
1966     }
1967 }
1968
1969 static av_always_inline void encode_mb_internal(MpegEncContext *s,
1970                                                 int motion_x, int motion_y,
1971                                                 int mb_block_height,
1972                                                 int mb_block_count)
1973 {
1974     int16_t weight[8][64];
1975     int16_t orig[8][64];
1976     const int mb_x = s->mb_x;
1977     const int mb_y = s->mb_y;
1978     int i;
1979     int skip_dct[8];
1980     int dct_offset = s->linesize * 8; // default for progressive frames
1981     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1982     ptrdiff_t wrap_y, wrap_c;
1983
1984     for (i = 0; i < mb_block_count; i++)
1985         skip_dct[i] = s->skipdct;
1986
1987     if (s->adaptive_quant) {
1988         const int last_qp = s->qscale;
1989         const int mb_xy = mb_x + mb_y * s->mb_stride;
1990
1991         s->lambda = s->lambda_table[mb_xy];
1992         update_qscale(s);
1993
1994         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
1995             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
1996             s->dquant = s->qscale - last_qp;
1997
1998             if (s->out_format == FMT_H263) {
1999                 s->dquant = av_clip(s->dquant, -2, 2);
2000
2001                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
2002                     if (!s->mb_intra) {
2003                         if (s->pict_type == AV_PICTURE_TYPE_B) {
2004                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
2005                                 s->dquant = 0;
2006                         }
2007                         if (s->mv_type == MV_TYPE_8X8)
2008                             s->dquant = 0;
2009                     }
2010                 }
2011             }
2012         }
2013         ff_set_qscale(s, last_qp + s->dquant);
2014     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
2015         ff_set_qscale(s, s->qscale + s->dquant);
2016
2017     wrap_y = s->linesize;
2018     wrap_c = s->uvlinesize;
2019     ptr_y  = s->new_picture.f->data[0] +
2020              (mb_y * 16 * wrap_y)              + mb_x * 16;
2021     ptr_cb = s->new_picture.f->data[1] +
2022              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
2023     ptr_cr = s->new_picture.f->data[2] +
2024              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
2025
2026     if (mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) {
2027         uint8_t *ebuf = s->sc.edge_emu_buffer + 32;
2028         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
2029                                  wrap_y, wrap_y,
2030                                  16, 16, mb_x * 16, mb_y * 16,
2031                                  s->width, s->height);
2032         ptr_y = ebuf;
2033         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y, ptr_cb,
2034                                  wrap_c, wrap_c,
2035                                  8, mb_block_height, mb_x * 8, mb_y * 8,
2036                                  s->width >> 1, s->height >> 1);
2037         ptr_cb = ebuf + 18 * wrap_y;
2038         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y + 8, ptr_cr,
2039                                  wrap_c, wrap_c,
2040                                  8, mb_block_height, mb_x * 8, mb_y * 8,
2041                                  s->width >> 1, s->height >> 1);
2042         ptr_cr = ebuf + 18 * wrap_y + 8;
2043     }
2044
2045     if (s->mb_intra) {
2046         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2047             int progressive_score, interlaced_score;
2048
2049             s->interlaced_dct = 0;
2050             progressive_score = s->mecc.ildct_cmp[4](s, ptr_y, NULL, wrap_y, 8) +
2051                                 s->mecc.ildct_cmp[4](s, ptr_y + wrap_y * 8,
2052                                                      NULL, wrap_y, 8) - 400;
2053
2054             if (progressive_score > 0) {
2055                 interlaced_score = s->mecc.ildct_cmp[4](s, ptr_y,
2056                                                         NULL, wrap_y * 2, 8) +
2057                                    s->mecc.ildct_cmp[4](s, ptr_y + wrap_y,
2058                                                         NULL, wrap_y * 2, 8);
2059                 if (progressive_score > interlaced_score) {
2060                     s->interlaced_dct = 1;
2061
2062                     dct_offset = wrap_y;
2063                     wrap_y <<= 1;
2064                     if (s->chroma_format == CHROMA_422)
2065                         wrap_c <<= 1;
2066                 }
2067             }
2068         }
2069
2070         s->pdsp.get_pixels(s->block[0], ptr_y,                  wrap_y);
2071         s->pdsp.get_pixels(s->block[1], ptr_y + 8,              wrap_y);
2072         s->pdsp.get_pixels(s->block[2], ptr_y + dct_offset,     wrap_y);
2073         s->pdsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
2074
2075         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2076             skip_dct[4] = 1;
2077             skip_dct[5] = 1;
2078         } else {
2079             s->pdsp.get_pixels(s->block[4], ptr_cb, wrap_c);
2080             s->pdsp.get_pixels(s->block[5], ptr_cr, wrap_c);
2081             if (!s->chroma_y_shift) { /* 422 */
2082                 s->pdsp.get_pixels(s->block[6],
2083                                    ptr_cb + (dct_offset >> 1), wrap_c);
2084                 s->pdsp.get_pixels(s->block[7],
2085                                    ptr_cr + (dct_offset >> 1), wrap_c);
2086             }
2087         }
2088     } else {
2089         op_pixels_func (*op_pix)[4];
2090         qpel_mc_func (*op_qpix)[16];
2091         uint8_t *dest_y, *dest_cb, *dest_cr;
2092
2093         dest_y  = s->dest[0];
2094         dest_cb = s->dest[1];
2095         dest_cr = s->dest[2];
2096
2097         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
2098             op_pix  = s->hdsp.put_pixels_tab;
2099             op_qpix = s->qdsp.put_qpel_pixels_tab;
2100         } else {
2101             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
2102             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
2103         }
2104
2105         if (s->mv_dir & MV_DIR_FORWARD) {
2106             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 0,
2107                           s->last_picture.f->data,
2108                           op_pix, op_qpix);
2109             op_pix  = s->hdsp.avg_pixels_tab;
2110             op_qpix = s->qdsp.avg_qpel_pixels_tab;
2111         }
2112         if (s->mv_dir & MV_DIR_BACKWARD) {
2113             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 1,
2114                           s->next_picture.f->data,
2115                           op_pix, op_qpix);
2116         }
2117
2118         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2119             int progressive_score, interlaced_score;
2120
2121             s->interlaced_dct = 0;
2122             progressive_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y, wrap_y, 8) +
2123                                 s->mecc.ildct_cmp[0](s, dest_y + wrap_y * 8,
2124                                                      ptr_y + wrap_y * 8,
2125                                                      wrap_y, 8) - 400;
2126
2127             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2128                 progressive_score -= 400;
2129
2130             if (progressive_score > 0) {
2131                 interlaced_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y,
2132                                                         wrap_y * 2, 8) +
2133                                    s->mecc.ildct_cmp[0](s, dest_y + wrap_y,
2134                                                         ptr_y + wrap_y,
2135                                                         wrap_y * 2, 8);
2136
2137                 if (progressive_score > interlaced_score) {
2138                     s->interlaced_dct = 1;
2139
2140                     dct_offset = wrap_y;
2141                     wrap_y <<= 1;
2142                     if (s->chroma_format == CHROMA_422)
2143                         wrap_c <<= 1;
2144                 }
2145             }
2146         }
2147
2148         s->pdsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2149         s->pdsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2150         s->pdsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2151                             dest_y + dct_offset, wrap_y);
2152         s->pdsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2153                             dest_y + dct_offset + 8, wrap_y);
2154
2155         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2156             skip_dct[4] = 1;
2157             skip_dct[5] = 1;
2158         } else {
2159             s->pdsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2160             s->pdsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2161             if (!s->chroma_y_shift) { /* 422 */
2162                 s->pdsp.diff_pixels(s->block[6], ptr_cb + (dct_offset >> 1),
2163                                     dest_cb + (dct_offset >> 1), wrap_c);
2164                 s->pdsp.diff_pixels(s->block[7], ptr_cr + (dct_offset >> 1),
2165                                     dest_cr + (dct_offset >> 1), wrap_c);
2166             }
2167         }
2168         /* pre quantization */
2169         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2170                 2 * s->qscale * s->qscale) {
2171             // FIXME optimize
2172             if (s->mecc.sad[1](NULL, ptr_y, dest_y, wrap_y, 8) < 20 * s->qscale)
2173                 skip_dct[0] = 1;
2174             if (s->mecc.sad[1](NULL, ptr_y + 8, dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2175                 skip_dct[1] = 1;
2176             if (s->mecc.sad[1](NULL, ptr_y + dct_offset, dest_y + dct_offset,
2177                                wrap_y, 8) < 20 * s->qscale)
2178                 skip_dct[2] = 1;
2179             if (s->mecc.sad[1](NULL, ptr_y + dct_offset + 8, dest_y + dct_offset + 8,
2180                                wrap_y, 8) < 20 * s->qscale)
2181                 skip_dct[3] = 1;
2182             if (s->mecc.sad[1](NULL, ptr_cb, dest_cb, wrap_c, 8) < 20 * s->qscale)
2183                 skip_dct[4] = 1;
2184             if (s->mecc.sad[1](NULL, ptr_cr, dest_cr, wrap_c, 8) < 20 * s->qscale)
2185                 skip_dct[5] = 1;
2186             if (!s->chroma_y_shift) { /* 422 */
2187                 if (s->mecc.sad[1](NULL, ptr_cb + (dct_offset >> 1),
2188                                    dest_cb + (dct_offset >> 1),
2189                                    wrap_c, 8) < 20 * s->qscale)
2190                     skip_dct[6] = 1;
2191                 if (s->mecc.sad[1](NULL, ptr_cr + (dct_offset >> 1),
2192                                    dest_cr + (dct_offset >> 1),
2193                                    wrap_c, 8) < 20 * s->qscale)
2194                     skip_dct[7] = 1;
2195             }
2196         }
2197     }
2198
2199     if (s->quantizer_noise_shaping) {
2200         if (!skip_dct[0])
2201             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2202         if (!skip_dct[1])
2203             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2204         if (!skip_dct[2])
2205             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2206         if (!skip_dct[3])
2207             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2208         if (!skip_dct[4])
2209             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2210         if (!skip_dct[5])
2211             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2212         if (!s->chroma_y_shift) { /* 422 */
2213             if (!skip_dct[6])
2214                 get_visual_weight(weight[6], ptr_cb + (dct_offset >> 1),
2215                                   wrap_c);
2216             if (!skip_dct[7])
2217                 get_visual_weight(weight[7], ptr_cr + (dct_offset >> 1),
2218                                   wrap_c);
2219         }
2220         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2221     }
2222
2223     /* DCT & quantize */
2224     assert(s->out_format != FMT_MJPEG || s->qscale == 8);
2225     {
2226         for (i = 0; i < mb_block_count; i++) {
2227             if (!skip_dct[i]) {
2228                 int overflow;
2229                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2230                 // FIXME we could decide to change to quantizer instead of
2231                 // clipping
2232                 // JS: I don't think that would be a good idea it could lower
2233                 //     quality instead of improve it. Just INTRADC clipping
2234                 //     deserves changes in quantizer
2235                 if (overflow)
2236                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2237             } else
2238                 s->block_last_index[i] = -1;
2239         }
2240         if (s->quantizer_noise_shaping) {
2241             for (i = 0; i < mb_block_count; i++) {
2242                 if (!skip_dct[i]) {
2243                     s->block_last_index[i] =
2244                         dct_quantize_refine(s, s->block[i], weight[i],
2245                                             orig[i], i, s->qscale);
2246                 }
2247             }
2248         }
2249
2250         if (s->luma_elim_threshold && !s->mb_intra)
2251             for (i = 0; i < 4; i++)
2252                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2253         if (s->chroma_elim_threshold && !s->mb_intra)
2254             for (i = 4; i < mb_block_count; i++)
2255                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2256
2257         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2258             for (i = 0; i < mb_block_count; i++) {
2259                 if (s->block_last_index[i] == -1)
2260                     s->coded_score[i] = INT_MAX / 256;
2261             }
2262         }
2263     }
2264
2265     if ((s->avctx->flags & AV_CODEC_FLAG_GRAY) && s->mb_intra) {
2266         s->block_last_index[4] =
2267         s->block_last_index[5] = 0;
2268         s->block[4][0] =
2269         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2270     }
2271
2272     // non c quantize code returns incorrect block_last_index FIXME
2273     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2274         for (i = 0; i < mb_block_count; i++) {
2275             int j;
2276             if (s->block_last_index[i] > 0) {
2277                 for (j = 63; j > 0; j--) {
2278                     if (s->block[i][s->intra_scantable.permutated[j]])
2279                         break;
2280                 }
2281                 s->block_last_index[i] = j;
2282             }
2283         }
2284     }
2285
2286     /* huffman encode */
2287     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2288     case AV_CODEC_ID_MPEG1VIDEO:
2289     case AV_CODEC_ID_MPEG2VIDEO:
2290         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2291             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2292         break;
2293     case AV_CODEC_ID_MPEG4:
2294         if (CONFIG_MPEG4_ENCODER)
2295             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2296         break;
2297     case AV_CODEC_ID_MSMPEG4V2:
2298     case AV_CODEC_ID_MSMPEG4V3:
2299     case AV_CODEC_ID_WMV1:
2300         if (CONFIG_MSMPEG4_ENCODER)
2301             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2302         break;
2303     case AV_CODEC_ID_WMV2:
2304         if (CONFIG_WMV2_ENCODER)
2305             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2306         break;
2307     case AV_CODEC_ID_H261:
2308         if (CONFIG_H261_ENCODER)
2309             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2310         break;
2311     case AV_CODEC_ID_H263:
2312     case AV_CODEC_ID_H263P:
2313     case AV_CODEC_ID_FLV1:
2314     case AV_CODEC_ID_RV10:
2315     case AV_CODEC_ID_RV20:
2316         if (CONFIG_H263_ENCODER)
2317             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2318         break;
2319     case AV_CODEC_ID_MJPEG:
2320         if (CONFIG_MJPEG_ENCODER)
2321             ff_mjpeg_encode_mb(s, s->block);
2322         break;
2323     default:
2324         assert(0);
2325     }
2326 }
2327
2328 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2329 {
2330     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
2331     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
2332 }
2333
2334 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2335     int i;
2336
2337     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2338
2339     /* mpeg1 */
2340     d->mb_skip_run= s->mb_skip_run;
2341     for(i=0; i<3; i++)
2342         d->last_dc[i] = s->last_dc[i];
2343
2344     /* statistics */
2345     d->mv_bits= s->mv_bits;
2346     d->i_tex_bits= s->i_tex_bits;
2347     d->p_tex_bits= s->p_tex_bits;
2348     d->i_count= s->i_count;
2349     d->f_count= s->f_count;
2350     d->b_count= s->b_count;
2351     d->skip_count= s->skip_count;
2352     d->misc_bits= s->misc_bits;
2353     d->last_bits= 0;
2354
2355     d->mb_skipped= 0;
2356     d->qscale= s->qscale;
2357     d->dquant= s->dquant;
2358
2359     d->esc3_level_length= s->esc3_level_length;
2360 }
2361
2362 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2363     int i;
2364
2365     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2366     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2367
2368     /* mpeg1 */
2369     d->mb_skip_run= s->mb_skip_run;
2370     for(i=0; i<3; i++)
2371         d->last_dc[i] = s->last_dc[i];
2372
2373     /* statistics */
2374     d->mv_bits= s->mv_bits;
2375     d->i_tex_bits= s->i_tex_bits;
2376     d->p_tex_bits= s->p_tex_bits;
2377     d->i_count= s->i_count;
2378     d->f_count= s->f_count;
2379     d->b_count= s->b_count;
2380     d->skip_count= s->skip_count;
2381     d->misc_bits= s->misc_bits;
2382
2383     d->mb_intra= s->mb_intra;
2384     d->mb_skipped= s->mb_skipped;
2385     d->mv_type= s->mv_type;
2386     d->mv_dir= s->mv_dir;
2387     d->pb= s->pb;
2388     if(s->data_partitioning){
2389         d->pb2= s->pb2;
2390         d->tex_pb= s->tex_pb;
2391     }
2392     d->block= s->block;
2393     for(i=0; i<8; i++)
2394         d->block_last_index[i]= s->block_last_index[i];
2395     d->interlaced_dct= s->interlaced_dct;
2396     d->qscale= s->qscale;
2397
2398     d->esc3_level_length= s->esc3_level_length;
2399 }
2400
2401 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2402                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2403                            int *dmin, int *next_block, int motion_x, int motion_y)
2404 {
2405     int score;
2406     uint8_t *dest_backup[3];
2407
2408     copy_context_before_encode(s, backup, type);
2409
2410     s->block= s->blocks[*next_block];
2411     s->pb= pb[*next_block];
2412     if(s->data_partitioning){
2413         s->pb2   = pb2   [*next_block];
2414         s->tex_pb= tex_pb[*next_block];
2415     }
2416
2417     if(*next_block){
2418         memcpy(dest_backup, s->dest, sizeof(s->dest));
2419         s->dest[0] = s->sc.rd_scratchpad;
2420         s->dest[1] = s->sc.rd_scratchpad + 16*s->linesize;
2421         s->dest[2] = s->sc.rd_scratchpad + 16*s->linesize + 8;
2422         assert(s->linesize >= 32); //FIXME
2423     }
2424
2425     encode_mb(s, motion_x, motion_y);
2426
2427     score= put_bits_count(&s->pb);
2428     if(s->data_partitioning){
2429         score+= put_bits_count(&s->pb2);
2430         score+= put_bits_count(&s->tex_pb);
2431     }
2432
2433     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2434         ff_mpv_decode_mb(s, s->block);
2435
2436         score *= s->lambda2;
2437         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2438     }
2439
2440     if(*next_block){
2441         memcpy(s->dest, dest_backup, sizeof(s->dest));
2442     }
2443
2444     if(score<*dmin){
2445         *dmin= score;
2446         *next_block^=1;
2447
2448         copy_context_after_encode(best, s, type);
2449     }
2450 }
2451
2452 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2453     uint32_t *sq = ff_square_tab + 256;
2454     int acc=0;
2455     int x,y;
2456
2457     if(w==16 && h==16)
2458         return s->mecc.sse[0](NULL, src1, src2, stride, 16);
2459     else if(w==8 && h==8)
2460         return s->mecc.sse[1](NULL, src1, src2, stride, 8);
2461
2462     for(y=0; y<h; y++){
2463         for(x=0; x<w; x++){
2464             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2465         }
2466     }
2467
2468     assert(acc>=0);
2469
2470     return acc;
2471 }
2472
2473 static int sse_mb(MpegEncContext *s){
2474     int w= 16;
2475     int h= 16;
2476
2477     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2478     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2479
2480     if(w==16 && h==16)
2481       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2482         return s->mecc.nsse[0](s, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2483                s->mecc.nsse[1](s, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2484                s->mecc.nsse[1](s, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2485       }else{
2486         return s->mecc.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2487                s->mecc.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2488                s->mecc.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2489       }
2490     else
2491         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2492                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2493                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2494 }
2495
2496 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2497     MpegEncContext *s= *(void**)arg;
2498
2499
2500     s->me.pre_pass=1;
2501     s->me.dia_size= s->avctx->pre_dia_size;
2502     s->first_slice_line=1;
2503     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2504         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2505             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2506         }
2507         s->first_slice_line=0;
2508     }
2509
2510     s->me.pre_pass=0;
2511
2512     return 0;
2513 }
2514
2515 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2516     MpegEncContext *s= *(void**)arg;
2517
2518     s->me.dia_size= s->avctx->dia_size;
2519     s->first_slice_line=1;
2520     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2521         s->mb_x=0; //for block init below
2522         ff_init_block_index(s);
2523         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2524             s->block_index[0]+=2;
2525             s->block_index[1]+=2;
2526             s->block_index[2]+=2;
2527             s->block_index[3]+=2;
2528
2529             /* compute motion vector & mb_type and store in context */
2530             if(s->pict_type==AV_PICTURE_TYPE_B)
2531                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2532             else
2533                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2534         }
2535         s->first_slice_line=0;
2536     }
2537     return 0;
2538 }
2539
2540 static int mb_var_thread(AVCodecContext *c, void *arg){
2541     MpegEncContext *s= *(void**)arg;
2542     int mb_x, mb_y;
2543
2544     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2545         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2546             int xx = mb_x * 16;
2547             int yy = mb_y * 16;
2548             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2549             int varc;
2550             int sum = s->mpvencdsp.pix_sum(pix, s->linesize);
2551
2552             varc = (s->mpvencdsp.pix_norm1(pix, s->linesize) -
2553                     (((unsigned) sum * sum) >> 8) + 500 + 128) >> 8;
2554
2555             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2556             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2557             s->me.mb_var_sum_temp    += varc;
2558         }
2559     }
2560     return 0;
2561 }
2562
2563 static void write_slice_end(MpegEncContext *s){
2564     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2565         if(s->partitioned_frame){
2566             ff_mpeg4_merge_partitions(s);
2567         }
2568
2569         ff_mpeg4_stuffing(&s->pb);
2570     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2571         ff_mjpeg_encode_stuffing(&s->pb);
2572     }
2573
2574     avpriv_align_put_bits(&s->pb);
2575     flush_put_bits(&s->pb);
2576
2577     if ((s->avctx->flags & AV_CODEC_FLAG_PASS1) && !s->partitioned_frame)
2578         s->misc_bits+= get_bits_diff(s);
2579 }
2580
2581 static void write_mb_info(MpegEncContext *s)
2582 {
2583     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2584     int offset = put_bits_count(&s->pb);
2585     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2586     int gobn = s->mb_y / s->gob_index;
2587     int pred_x, pred_y;
2588     if (CONFIG_H263_ENCODER)
2589         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2590     bytestream_put_le32(&ptr, offset);
2591     bytestream_put_byte(&ptr, s->qscale);
2592     bytestream_put_byte(&ptr, gobn);
2593     bytestream_put_le16(&ptr, mba);
2594     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2595     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2596     /* 4MV not implemented */
2597     bytestream_put_byte(&ptr, 0); /* hmv2 */
2598     bytestream_put_byte(&ptr, 0); /* vmv2 */
2599 }
2600
2601 static void update_mb_info(MpegEncContext *s, int startcode)
2602 {
2603     if (!s->mb_info)
2604         return;
2605     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2606         s->mb_info_size += 12;
2607         s->prev_mb_info = s->last_mb_info;
2608     }
2609     if (startcode) {
2610         s->prev_mb_info = put_bits_count(&s->pb)/8;
2611         /* This might have incremented mb_info_size above, and we return without
2612          * actually writing any info into that slot yet. But in that case,
2613          * this will be called again at the start of the after writing the
2614          * start code, actually writing the mb info. */
2615         return;
2616     }
2617
2618     s->last_mb_info = put_bits_count(&s->pb)/8;
2619     if (!s->mb_info_size)
2620         s->mb_info_size += 12;
2621     write_mb_info(s);
2622 }
2623
2624 static int encode_thread(AVCodecContext *c, void *arg){
2625     MpegEncContext *s= *(void**)arg;
2626     int mb_x, mb_y, pdif = 0;
2627     int chr_h= 16>>s->chroma_y_shift;
2628     int i, j;
2629     MpegEncContext best_s = { 0 }, backup_s;
2630     uint8_t bit_buf[2][MAX_MB_BYTES];
2631     uint8_t bit_buf2[2][MAX_MB_BYTES];
2632     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2633     PutBitContext pb[2], pb2[2], tex_pb[2];
2634
2635     for(i=0; i<2; i++){
2636         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2637         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2638         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2639     }
2640
2641     s->last_bits= put_bits_count(&s->pb);
2642     s->mv_bits=0;
2643     s->misc_bits=0;
2644     s->i_tex_bits=0;
2645     s->p_tex_bits=0;
2646     s->i_count=0;
2647     s->f_count=0;
2648     s->b_count=0;
2649     s->skip_count=0;
2650
2651     for(i=0; i<3; i++){
2652         /* init last dc values */
2653         /* note: quant matrix value (8) is implied here */
2654         s->last_dc[i] = 128 << s->intra_dc_precision;
2655
2656         s->current_picture.encoding_error[i] = 0;
2657     }
2658     s->mb_skip_run = 0;
2659     memset(s->last_mv, 0, sizeof(s->last_mv));
2660
2661     s->last_mv_dir = 0;
2662
2663     switch(s->codec_id){
2664     case AV_CODEC_ID_H263:
2665     case AV_CODEC_ID_H263P:
2666     case AV_CODEC_ID_FLV1:
2667         if (CONFIG_H263_ENCODER)
2668             s->gob_index = H263_GOB_HEIGHT(s->height);
2669         break;
2670     case AV_CODEC_ID_MPEG4:
2671         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2672             ff_mpeg4_init_partitions(s);
2673         break;
2674     }
2675
2676     s->resync_mb_x=0;
2677     s->resync_mb_y=0;
2678     s->first_slice_line = 1;
2679     s->ptr_lastgob = s->pb.buf;
2680     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2681         s->mb_x=0;
2682         s->mb_y= mb_y;
2683
2684         ff_set_qscale(s, s->qscale);
2685         ff_init_block_index(s);
2686
2687         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2688             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2689             int mb_type= s->mb_type[xy];
2690 //            int d;
2691             int dmin= INT_MAX;
2692             int dir;
2693
2694             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2695                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2696                 return -1;
2697             }
2698             if(s->data_partitioning){
2699                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2700                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2701                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2702                     return -1;
2703                 }
2704             }
2705
2706             s->mb_x = mb_x;
2707             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2708             ff_update_block_index(s);
2709
2710             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
2711                 ff_h261_reorder_mb_index(s);
2712                 xy= s->mb_y*s->mb_stride + s->mb_x;
2713                 mb_type= s->mb_type[xy];
2714             }
2715
2716             /* write gob / video packet header  */
2717             if(s->rtp_mode){
2718                 int current_packet_size, is_gob_start;
2719
2720                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2721
2722                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2723
2724                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2725
2726                 switch(s->codec_id){
2727                 case AV_CODEC_ID_H263:
2728                 case AV_CODEC_ID_H263P:
2729                     if(!s->h263_slice_structured)
2730                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2731                     break;
2732                 case AV_CODEC_ID_MPEG2VIDEO:
2733                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2734                 case AV_CODEC_ID_MPEG1VIDEO:
2735                     if(s->mb_skip_run) is_gob_start=0;
2736                     break;
2737                 }
2738
2739                 if(is_gob_start){
2740                     if(s->start_mb_y != mb_y || mb_x!=0){
2741                         write_slice_end(s);
2742
2743                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
2744                             ff_mpeg4_init_partitions(s);
2745                         }
2746                     }
2747
2748                     assert((put_bits_count(&s->pb)&7) == 0);
2749                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2750
2751                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
2752                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2753                         int d = 100 / s->error_rate;
2754                         if(r % d == 0){
2755                             current_packet_size=0;
2756                             s->pb.buf_ptr= s->ptr_lastgob;
2757                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2758                         }
2759                     }
2760
2761 #if FF_API_RTP_CALLBACK
2762 FF_DISABLE_DEPRECATION_WARNINGS
2763                     if (s->avctx->rtp_callback){
2764                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2765                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2766                     }
2767 FF_ENABLE_DEPRECATION_WARNINGS
2768 #endif
2769                     update_mb_info(s, 1);
2770
2771                     switch(s->codec_id){
2772                     case AV_CODEC_ID_MPEG4:
2773                         if (CONFIG_MPEG4_ENCODER) {
2774                             ff_mpeg4_encode_video_packet_header(s);
2775                             ff_mpeg4_clean_buffers(s);
2776                         }
2777                     break;
2778                     case AV_CODEC_ID_MPEG1VIDEO:
2779                     case AV_CODEC_ID_MPEG2VIDEO:
2780                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2781                             ff_mpeg1_encode_slice_header(s);
2782                             ff_mpeg1_clean_buffers(s);
2783                         }
2784                     break;
2785                     case AV_CODEC_ID_H263:
2786                     case AV_CODEC_ID_H263P:
2787                         if (CONFIG_H263_ENCODER)
2788                             ff_h263_encode_gob_header(s, mb_y);
2789                     break;
2790                     }
2791
2792                     if (s->avctx->flags & AV_CODEC_FLAG_PASS1) {
2793                         int bits= put_bits_count(&s->pb);
2794                         s->misc_bits+= bits - s->last_bits;
2795                         s->last_bits= bits;
2796                     }
2797
2798                     s->ptr_lastgob += current_packet_size;
2799                     s->first_slice_line=1;
2800                     s->resync_mb_x=mb_x;
2801                     s->resync_mb_y=mb_y;
2802                 }
2803             }
2804
2805             if(  (s->resync_mb_x   == s->mb_x)
2806                && s->resync_mb_y+1 == s->mb_y){
2807                 s->first_slice_line=0;
2808             }
2809
2810             s->mb_skipped=0;
2811             s->dquant=0; //only for QP_RD
2812
2813             update_mb_info(s, 0);
2814
2815             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
2816                 int next_block=0;
2817                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2818
2819                 copy_context_before_encode(&backup_s, s, -1);
2820                 backup_s.pb= s->pb;
2821                 best_s.data_partitioning= s->data_partitioning;
2822                 best_s.partitioned_frame= s->partitioned_frame;
2823                 if(s->data_partitioning){
2824                     backup_s.pb2= s->pb2;
2825                     backup_s.tex_pb= s->tex_pb;
2826                 }
2827
2828                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2829                     s->mv_dir = MV_DIR_FORWARD;
2830                     s->mv_type = MV_TYPE_16X16;
2831                     s->mb_intra= 0;
2832                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2833                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2834                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2835                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2836                 }
2837                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2838                     s->mv_dir = MV_DIR_FORWARD;
2839                     s->mv_type = MV_TYPE_FIELD;
2840                     s->mb_intra= 0;
2841                     for(i=0; i<2; i++){
2842                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2843                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2844                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2845                     }
2846                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2847                                  &dmin, &next_block, 0, 0);
2848                 }
2849                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2850                     s->mv_dir = MV_DIR_FORWARD;
2851                     s->mv_type = MV_TYPE_16X16;
2852                     s->mb_intra= 0;
2853                     s->mv[0][0][0] = 0;
2854                     s->mv[0][0][1] = 0;
2855                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2856                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2857                 }
2858                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2859                     s->mv_dir = MV_DIR_FORWARD;
2860                     s->mv_type = MV_TYPE_8X8;
2861                     s->mb_intra= 0;
2862                     for(i=0; i<4; i++){
2863                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2864                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2865                     }
2866                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2867                                  &dmin, &next_block, 0, 0);
2868                 }
2869                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2870                     s->mv_dir = MV_DIR_FORWARD;
2871                     s->mv_type = MV_TYPE_16X16;
2872                     s->mb_intra= 0;
2873                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2874                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2875                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2876                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2877                 }
2878                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2879                     s->mv_dir = MV_DIR_BACKWARD;
2880                     s->mv_type = MV_TYPE_16X16;
2881                     s->mb_intra= 0;
2882                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2883                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2884                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2885                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2886                 }
2887                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2888                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2889                     s->mv_type = MV_TYPE_16X16;
2890                     s->mb_intra= 0;
2891                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2892                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2893                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2894                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2895                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2896                                  &dmin, &next_block, 0, 0);
2897                 }
2898                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2899                     s->mv_dir = MV_DIR_FORWARD;
2900                     s->mv_type = MV_TYPE_FIELD;
2901                     s->mb_intra= 0;
2902                     for(i=0; i<2; i++){
2903                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2904                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2905                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2906                     }
2907                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2908                                  &dmin, &next_block, 0, 0);
2909                 }
2910                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2911                     s->mv_dir = MV_DIR_BACKWARD;
2912                     s->mv_type = MV_TYPE_FIELD;
2913                     s->mb_intra= 0;
2914                     for(i=0; i<2; i++){
2915                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2916                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2917                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2918                     }
2919                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2920                                  &dmin, &next_block, 0, 0);
2921                 }
2922                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2923                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2924                     s->mv_type = MV_TYPE_FIELD;
2925                     s->mb_intra= 0;
2926                     for(dir=0; dir<2; dir++){
2927                         for(i=0; i<2; i++){
2928                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2929                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2930                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2931                         }
2932                     }
2933                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2934                                  &dmin, &next_block, 0, 0);
2935                 }
2936                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2937                     s->mv_dir = 0;
2938                     s->mv_type = MV_TYPE_16X16;
2939                     s->mb_intra= 1;
2940                     s->mv[0][0][0] = 0;
2941                     s->mv[0][0][1] = 0;
2942                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2943                                  &dmin, &next_block, 0, 0);
2944                     if(s->h263_pred || s->h263_aic){
2945                         if(best_s.mb_intra)
2946                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2947                         else
2948                             ff_clean_intra_table_entries(s); //old mode?
2949                     }
2950                 }
2951
2952                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
2953                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2954                         const int last_qp= backup_s.qscale;
2955                         int qpi, qp, dc[6];
2956                         int16_t ac[6][16];
2957                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2958                         static const int dquant_tab[4]={-1,1,-2,2};
2959
2960                         assert(backup_s.dquant == 0);
2961
2962                         //FIXME intra
2963                         s->mv_dir= best_s.mv_dir;
2964                         s->mv_type = MV_TYPE_16X16;
2965                         s->mb_intra= best_s.mb_intra;
2966                         s->mv[0][0][0] = best_s.mv[0][0][0];
2967                         s->mv[0][0][1] = best_s.mv[0][0][1];
2968                         s->mv[1][0][0] = best_s.mv[1][0][0];
2969                         s->mv[1][0][1] = best_s.mv[1][0][1];
2970
2971                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
2972                         for(; qpi<4; qpi++){
2973                             int dquant= dquant_tab[qpi];
2974                             qp= last_qp + dquant;
2975                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2976                                 continue;
2977                             backup_s.dquant= dquant;
2978                             if(s->mb_intra && s->dc_val[0]){
2979                                 for(i=0; i<6; i++){
2980                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2981                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(int16_t)*16);
2982                                 }
2983                             }
2984
2985                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2986                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2987                             if(best_s.qscale != qp){
2988                                 if(s->mb_intra && s->dc_val[0]){
2989                                     for(i=0; i<6; i++){
2990                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2991                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(int16_t)*16);
2992                                     }
2993                                 }
2994                             }
2995                         }
2996                     }
2997                 }
2998                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2999                     int mx= s->b_direct_mv_table[xy][0];
3000                     int my= s->b_direct_mv_table[xy][1];
3001
3002                     backup_s.dquant = 0;
3003                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3004                     s->mb_intra= 0;
3005                     ff_mpeg4_set_direct_mv(s, mx, my);
3006                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3007                                  &dmin, &next_block, mx, my);
3008                 }
3009                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
3010                     backup_s.dquant = 0;
3011                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3012                     s->mb_intra= 0;
3013                     ff_mpeg4_set_direct_mv(s, 0, 0);
3014                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3015                                  &dmin, &next_block, 0, 0);
3016                 }
3017                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
3018                     int coded=0;
3019                     for(i=0; i<6; i++)
3020                         coded |= s->block_last_index[i];
3021                     if(coded){
3022                         int mx,my;
3023                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
3024                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
3025                             mx=my=0; //FIXME find the one we actually used
3026                             ff_mpeg4_set_direct_mv(s, mx, my);
3027                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
3028                             mx= s->mv[1][0][0];
3029                             my= s->mv[1][0][1];
3030                         }else{
3031                             mx= s->mv[0][0][0];
3032                             my= s->mv[0][0][1];
3033                         }
3034
3035                         s->mv_dir= best_s.mv_dir;
3036                         s->mv_type = best_s.mv_type;
3037                         s->mb_intra= 0;
3038 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
3039                         s->mv[0][0][1] = best_s.mv[0][0][1];
3040                         s->mv[1][0][0] = best_s.mv[1][0][0];
3041                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
3042                         backup_s.dquant= 0;
3043                         s->skipdct=1;
3044                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3045                                         &dmin, &next_block, mx, my);
3046                         s->skipdct=0;
3047                     }
3048                 }
3049
3050                 s->current_picture.qscale_table[xy] = best_s.qscale;
3051
3052                 copy_context_after_encode(s, &best_s, -1);
3053
3054                 pb_bits_count= put_bits_count(&s->pb);
3055                 flush_put_bits(&s->pb);
3056                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
3057                 s->pb= backup_s.pb;
3058
3059                 if(s->data_partitioning){
3060                     pb2_bits_count= put_bits_count(&s->pb2);
3061                     flush_put_bits(&s->pb2);
3062                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
3063                     s->pb2= backup_s.pb2;
3064
3065                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
3066                     flush_put_bits(&s->tex_pb);
3067                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
3068                     s->tex_pb= backup_s.tex_pb;
3069                 }
3070                 s->last_bits= put_bits_count(&s->pb);
3071
3072                 if (CONFIG_H263_ENCODER &&
3073                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3074                     ff_h263_update_motion_val(s);
3075
3076                 if(next_block==0){ //FIXME 16 vs linesize16
3077                     s->hdsp.put_pixels_tab[0][0](s->dest[0], s->sc.rd_scratchpad                     , s->linesize  ,16);
3078                     s->hdsp.put_pixels_tab[1][0](s->dest[1], s->sc.rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
3079                     s->hdsp.put_pixels_tab[1][0](s->dest[2], s->sc.rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
3080                 }
3081
3082                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
3083                     ff_mpv_decode_mb(s, s->block);
3084             } else {
3085                 int motion_x = 0, motion_y = 0;
3086                 s->mv_type=MV_TYPE_16X16;
3087                 // only one MB-Type possible
3088
3089                 switch(mb_type){
3090                 case CANDIDATE_MB_TYPE_INTRA:
3091                     s->mv_dir = 0;
3092                     s->mb_intra= 1;
3093                     motion_x= s->mv[0][0][0] = 0;
3094                     motion_y= s->mv[0][0][1] = 0;
3095                     break;
3096                 case CANDIDATE_MB_TYPE_INTER:
3097                     s->mv_dir = MV_DIR_FORWARD;
3098                     s->mb_intra= 0;
3099                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
3100                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
3101                     break;
3102                 case CANDIDATE_MB_TYPE_INTER_I:
3103                     s->mv_dir = MV_DIR_FORWARD;
3104                     s->mv_type = MV_TYPE_FIELD;
3105                     s->mb_intra= 0;
3106                     for(i=0; i<2; i++){
3107                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3108                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3109                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3110                     }
3111                     break;
3112                 case CANDIDATE_MB_TYPE_INTER4V:
3113                     s->mv_dir = MV_DIR_FORWARD;
3114                     s->mv_type = MV_TYPE_8X8;
3115                     s->mb_intra= 0;
3116                     for(i=0; i<4; i++){
3117                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3118                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3119                     }
3120                     break;
3121                 case CANDIDATE_MB_TYPE_DIRECT:
3122                     if (CONFIG_MPEG4_ENCODER) {
3123                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3124                         s->mb_intra= 0;
3125                         motion_x=s->b_direct_mv_table[xy][0];
3126                         motion_y=s->b_direct_mv_table[xy][1];
3127                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
3128                     }
3129                     break;
3130                 case CANDIDATE_MB_TYPE_DIRECT0:
3131                     if (CONFIG_MPEG4_ENCODER) {
3132                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3133                         s->mb_intra= 0;
3134                         ff_mpeg4_set_direct_mv(s, 0, 0);
3135                     }
3136                     break;
3137                 case CANDIDATE_MB_TYPE_BIDIR:
3138                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3139                     s->mb_intra= 0;
3140                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3141                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3142                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3143                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3144                     break;
3145                 case CANDIDATE_MB_TYPE_BACKWARD:
3146                     s->mv_dir = MV_DIR_BACKWARD;
3147                     s->mb_intra= 0;
3148                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3149                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3150                     break;
3151                 case CANDIDATE_MB_TYPE_FORWARD:
3152                     s->mv_dir = MV_DIR_FORWARD;
3153                     s->mb_intra= 0;
3154                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3155                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3156                     break;
3157                 case CANDIDATE_MB_TYPE_FORWARD_I:
3158                     s->mv_dir = MV_DIR_FORWARD;
3159                     s->mv_type = MV_TYPE_FIELD;
3160                     s->mb_intra= 0;
3161                     for(i=0; i<2; i++){
3162                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3163                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3164                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3165                     }
3166                     break;
3167                 case CANDIDATE_MB_TYPE_BACKWARD_I:
3168                     s->mv_dir = MV_DIR_BACKWARD;
3169                     s->mv_type = MV_TYPE_FIELD;
3170                     s->mb_intra= 0;
3171                     for(i=0; i<2; i++){
3172                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3173                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3174                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3175                     }
3176                     break;
3177                 case CANDIDATE_MB_TYPE_BIDIR_I:
3178                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3179                     s->mv_type = MV_TYPE_FIELD;
3180                     s->mb_intra= 0;
3181                     for(dir=0; dir<2; dir++){
3182                         for(i=0; i<2; i++){
3183                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3184                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3185                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3186                         }
3187                     }
3188                     break;
3189                 default:
3190                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
3191                 }
3192
3193                 encode_mb(s, motion_x, motion_y);
3194
3195                 // RAL: Update last macroblock type
3196                 s->last_mv_dir = s->mv_dir;
3197
3198                 if (CONFIG_H263_ENCODER &&
3199                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3200                     ff_h263_update_motion_val(s);
3201
3202                 ff_mpv_decode_mb(s, s->block);
3203             }
3204
3205             /* clean the MV table in IPS frames for direct mode in B frames */
3206             if(s->mb_intra /* && I,P,S_TYPE */){
3207                 s->p_mv_table[xy][0]=0;
3208                 s->p_mv_table[xy][1]=0;
3209             }
3210
3211             if (s->avctx->flags & AV_CODEC_FLAG_PSNR) {
3212                 int w= 16;
3213                 int h= 16;
3214
3215                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
3216                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
3217
3218                 s->current_picture.encoding_error[0] += sse(
3219                     s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
3220                     s->dest[0], w, h, s->linesize);
3221                 s->current_picture.encoding_error[1] += sse(
3222                     s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3223                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3224                 s->current_picture.encoding_error[2] += sse(
3225                     s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3226                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3227             }
3228             if(s->loop_filter){
3229                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
3230                     ff_h263_loop_filter(s);
3231             }
3232             ff_dlog(s->avctx, "MB %d %d bits\n",
3233                     s->mb_x + s->mb_y * s->mb_stride, put_bits_count(&s->pb));
3234         }
3235     }
3236
3237     //not beautiful here but we must write it before flushing so it has to be here
3238     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
3239         ff_msmpeg4_encode_ext_header(s);
3240
3241     write_slice_end(s);
3242
3243 #if FF_API_RTP_CALLBACK
3244 FF_DISABLE_DEPRECATION_WARNINGS
3245     /* Send the last GOB if RTP */
3246     if (s->avctx->rtp_callback) {
3247         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
3248         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
3249         /* Call the RTP callback to send the last GOB */
3250         emms_c();
3251         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
3252     }
3253 FF_ENABLE_DEPRECATION_WARNINGS
3254 #endif
3255
3256     return 0;
3257 }
3258
3259 #define MERGE(field) dst->field += src->field; src->field=0
3260 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
3261     MERGE(me.scene_change_score);
3262     MERGE(me.mc_mb_var_sum_temp);
3263     MERGE(me.mb_var_sum_temp);
3264 }
3265
3266 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
3267     int i;
3268
3269     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
3270     MERGE(dct_count[1]);
3271     MERGE(mv_bits);
3272     MERGE(i_tex_bits);
3273     MERGE(p_tex_bits);
3274     MERGE(i_count);
3275     MERGE(f_count);
3276     MERGE(b_count);
3277     MERGE(skip_count);
3278     MERGE(misc_bits);
3279     MERGE(er.error_count);
3280     MERGE(padding_bug_score);
3281     MERGE(current_picture.encoding_error[0]);
3282     MERGE(current_picture.encoding_error[1]);
3283     MERGE(current_picture.encoding_error[2]);
3284
3285     if(dst->avctx->noise_reduction){
3286         for(i=0; i<64; i++){
3287             MERGE(dct_error_sum[0][i]);
3288             MERGE(dct_error_sum[1][i]);
3289         }
3290     }
3291
3292     assert(put_bits_count(&src->pb) % 8 ==0);
3293     assert(put_bits_count(&dst->pb) % 8 ==0);
3294     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3295     flush_put_bits(&dst->pb);
3296 }
3297
3298 static int estimate_qp(MpegEncContext *s, int dry_run){
3299     if (s->next_lambda){
3300         s->current_picture_ptr->f->quality =
3301         s->current_picture.f->quality = s->next_lambda;
3302         if(!dry_run) s->next_lambda= 0;
3303     } else if (!s->fixed_qscale) {
3304         s->current_picture_ptr->f->quality =
3305         s->current_picture.f->quality = ff_rate_estimate_qscale(s, dry_run);
3306         if (s->current_picture.f->quality < 0)
3307             return -1;
3308     }
3309
3310     if(s->adaptive_quant){
3311         switch(s->codec_id){
3312         case AV_CODEC_ID_MPEG4:
3313             if (CONFIG_MPEG4_ENCODER)
3314                 ff_clean_mpeg4_qscales(s);
3315             break;
3316         case AV_CODEC_ID_H263:
3317         case AV_CODEC_ID_H263P:
3318         case AV_CODEC_ID_FLV1:
3319             if (CONFIG_H263_ENCODER)
3320                 ff_clean_h263_qscales(s);
3321             break;
3322         default:
3323             ff_init_qscale_tab(s);
3324         }
3325
3326         s->lambda= s->lambda_table[0];
3327         //FIXME broken
3328     }else
3329         s->lambda = s->current_picture.f->quality;
3330     update_qscale(s);
3331     return 0;
3332 }
3333
3334 /* must be called before writing the header */
3335 static void set_frame_distances(MpegEncContext * s){
3336     assert(s->current_picture_ptr->f->pts != AV_NOPTS_VALUE);
3337     s->time = s->current_picture_ptr->f->pts * s->avctx->time_base.num;
3338
3339     if(s->pict_type==AV_PICTURE_TYPE_B){
3340         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3341         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3342     }else{
3343         s->pp_time= s->time - s->last_non_b_time;
3344         s->last_non_b_time= s->time;
3345         assert(s->picture_number==0 || s->pp_time > 0);
3346     }
3347 }
3348
3349 static int encode_picture(MpegEncContext *s, int picture_number)
3350 {
3351     int i, ret;
3352     int bits;
3353     int context_count = s->slice_context_count;
3354
3355     s->picture_number = picture_number;
3356
3357     /* Reset the average MB variance */
3358     s->me.mb_var_sum_temp    =
3359     s->me.mc_mb_var_sum_temp = 0;
3360
3361     /* we need to initialize some time vars before we can encode b-frames */
3362     // RAL: Condition added for MPEG1VIDEO
3363     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3364         set_frame_distances(s);
3365     if(CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4)
3366         ff_set_mpeg4_time(s);
3367
3368     s->me.scene_change_score=0;
3369
3370 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3371
3372     if(s->pict_type==AV_PICTURE_TYPE_I){
3373         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3374         else                        s->no_rounding=0;
3375     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3376         if(s->flipflop_rounding || s->codec_id == AV_CODEC_ID_H263P || s->codec_id == AV_CODEC_ID_MPEG4)
3377             s->no_rounding ^= 1;
3378     }
3379
3380     if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
3381         if (estimate_qp(s,1) < 0)
3382             return -1;
3383         ff_get_2pass_fcode(s);
3384     } else if (!(s->avctx->flags & AV_CODEC_FLAG_QSCALE)) {
3385         if(s->pict_type==AV_PICTURE_TYPE_B)
3386             s->lambda= s->last_lambda_for[s->pict_type];
3387         else
3388             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3389         update_qscale(s);
3390     }
3391
3392     s->mb_intra=0; //for the rate distortion & bit compare functions
3393     for(i=1; i<context_count; i++){
3394         ret = ff_update_duplicate_context(s->thread_context[i], s);
3395         if (ret < 0)
3396             return ret;
3397     }
3398
3399     if(ff_init_me(s)<0)
3400         return -1;
3401
3402     /* Estimate motion for every MB */
3403     if(s->pict_type != AV_PICTURE_TYPE_I){
3404         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
3405         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
3406         if (s->pict_type != AV_PICTURE_TYPE_B) {
3407             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
3408                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3409             }
3410         }
3411
3412         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3413     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3414         /* I-Frame */
3415         for(i=0; i<s->mb_stride*s->mb_height; i++)
3416             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3417
3418         if(!s->fixed_qscale){
3419             /* finding spatial complexity for I-frame rate control */
3420             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3421         }
3422     }
3423     for(i=1; i<context_count; i++){
3424         merge_context_after_me(s, s->thread_context[i]);
3425     }
3426     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3427     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3428     emms_c();
3429
3430     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
3431         s->pict_type= AV_PICTURE_TYPE_I;
3432         for(i=0; i<s->mb_stride*s->mb_height; i++)
3433             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3434         ff_dlog(s, "Scene change detected, encoding as I Frame %d %d\n",
3435                 s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3436     }
3437
3438     if(!s->umvplus){
3439         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3440             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3441
3442             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3443                 int a,b;
3444                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3445                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3446                 s->f_code= FFMAX3(s->f_code, a, b);
3447             }
3448
3449             ff_fix_long_p_mvs(s);
3450             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3451             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3452                 int j;
3453                 for(i=0; i<2; i++){
3454                     for(j=0; j<2; j++)
3455                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3456                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3457                 }
3458             }
3459         }
3460
3461         if(s->pict_type==AV_PICTURE_TYPE_B){
3462             int a, b;
3463
3464             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3465             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3466             s->f_code = FFMAX(a, b);
3467
3468             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3469             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3470             s->b_code = FFMAX(a, b);
3471
3472             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3473             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3474             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3475             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3476             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3477                 int dir, j;
3478                 for(dir=0; dir<2; dir++){
3479                     for(i=0; i<2; i++){
3480                         for(j=0; j<2; j++){
3481                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3482                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3483                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3484                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3485                         }
3486                     }
3487                 }
3488             }
3489         }
3490     }
3491
3492     if (estimate_qp(s, 0) < 0)
3493         return -1;
3494
3495     if (s->qscale < 3 && s->max_qcoeff <= 128 &&
3496         s->pict_type == AV_PICTURE_TYPE_I &&
3497         !(s->avctx->flags & AV_CODEC_FLAG_QSCALE))
3498         s->qscale= 3; //reduce clipping problems
3499
3500     if (s->out_format == FMT_MJPEG) {
3501         /* for mjpeg, we do include qscale in the matrix */
3502         for(i=1;i<64;i++){
3503             int j = s->idsp.idct_permutation[i];
3504
3505             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
3506         }
3507         s->y_dc_scale_table=
3508         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3509         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3510         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
3511                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3512         s->qscale= 8;
3513     }
3514
3515     //FIXME var duplication
3516     s->current_picture_ptr->f->key_frame =
3517     s->current_picture.f->key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3518     s->current_picture_ptr->f->pict_type =
3519     s->current_picture.f->pict_type = s->pict_type;
3520
3521     if (s->current_picture.f->key_frame)
3522         s->picture_in_gop_number=0;
3523
3524     s->last_bits= put_bits_count(&s->pb);
3525     switch(s->out_format) {
3526     case FMT_MJPEG:
3527         if (CONFIG_MJPEG_ENCODER)
3528             ff_mjpeg_encode_picture_header(s->avctx, &s->pb, &s->intra_scantable,
3529                                            s->intra_matrix);
3530         break;
3531     case FMT_H261:
3532         if (CONFIG_H261_ENCODER)
3533             ff_h261_encode_picture_header(s, picture_number);
3534         break;
3535     case FMT_H263:
3536         if (CONFIG_WMV2_ENCODER && s->codec_id == AV_CODEC_ID_WMV2)
3537             ff_wmv2_encode_picture_header(s, picture_number);
3538         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3539             ff_msmpeg4_encode_picture_header(s, picture_number);
3540         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
3541             ff_mpeg4_encode_picture_header(s, picture_number);
3542         else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10) {
3543             ret = ff_rv10_encode_picture_header(s, picture_number);
3544             if (ret < 0)
3545                 return ret;
3546         }
3547         else if (CONFIG_RV20_ENCODER && s->codec_id == AV_CODEC_ID_RV20)
3548             ff_rv20_encode_picture_header(s, picture_number);
3549         else if (CONFIG_FLV_ENCODER && s->codec_id == AV_CODEC_ID_FLV1)
3550             ff_flv_encode_picture_header(s, picture_number);
3551         else if (CONFIG_H263_ENCODER)
3552             ff_h263_encode_picture_header(s, picture_number);
3553         break;
3554     case FMT_MPEG1:
3555         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
3556             ff_mpeg1_encode_picture_header(s, picture_number);
3557         break;
3558     default:
3559         assert(0);
3560     }
3561     bits= put_bits_count(&s->pb);
3562     s->header_bits= bits - s->last_bits;
3563
3564     for(i=1; i<context_count; i++){
3565         update_duplicate_context_after_me(s->thread_context[i], s);
3566     }
3567     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3568     for(i=1; i<context_count; i++){
3569         merge_context_after_encode(s, s->thread_context[i]);
3570     }
3571     emms_c();
3572     return 0;