lavc/htmlsubtitles: improve line breaks handling
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of FFmpeg.
9  *
10  * FFmpeg is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * FFmpeg is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with FFmpeg; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /*
26  * non linear quantizers with large QPs and VBV with restrictive qmin fixes sponsored by NOA GmbH
27  */
28
29 /**
30  * @file
31  * The simplest mpeg encoder (well, it was the simplest!).
32  */
33
34 #include <stdint.h>
35
36 #include "libavutil/internal.h"
37 #include "libavutil/intmath.h"
38 #include "libavutil/mathematics.h"
39 #include "libavutil/pixdesc.h"
40 #include "libavutil/opt.h"
41 #include "libavutil/timer.h"
42 #include "avcodec.h"
43 #include "dct.h"
44 #include "idctdsp.h"
45 #include "mpeg12.h"
46 #include "mpegvideo.h"
47 #include "mpegvideodata.h"
48 #include "h261.h"
49 #include "h263.h"
50 #include "h263data.h"
51 #include "mjpegenc_common.h"
52 #include "mathops.h"
53 #include "mpegutils.h"
54 #include "mjpegenc.h"
55 #include "msmpeg4.h"
56 #include "pixblockdsp.h"
57 #include "qpeldsp.h"
58 #include "faandct.h"
59 #include "thread.h"
60 #include "aandcttab.h"
61 #include "flv.h"
62 #include "mpeg4video.h"
63 #include "internal.h"
64 #include "bytestream.h"
65 #include "wmv2.h"
66 #include "rv10.h"
67 #include "libxvid.h"
68 #include <limits.h>
69 #include "sp5x.h"
70
71 #define QUANT_BIAS_SHIFT 8
72
73 #define QMAT_SHIFT_MMX 16
74 #define QMAT_SHIFT 21
75
76 static int encode_picture(MpegEncContext *s, int picture_number);
77 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
78 static int sse_mb(MpegEncContext *s);
79 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
80 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
81
82 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
83 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
84
85 const AVOption ff_mpv_generic_options[] = {
86     FF_MPV_COMMON_OPTS
87     { NULL },
88 };
89
90 void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64],
91                        uint16_t (*qmat16)[2][64],
92                        const uint16_t *quant_matrix,
93                        int bias, int qmin, int qmax, int intra)
94 {
95     FDCTDSPContext *fdsp = &s->fdsp;
96     int qscale;
97     int shift = 0;
98
99     for (qscale = qmin; qscale <= qmax; qscale++) {
100         int i;
101         int qscale2;
102
103         if (s->q_scale_type) qscale2 = ff_mpeg2_non_linear_qscale[qscale];
104         else                 qscale2 = qscale << 1;
105
106         if (fdsp->fdct == ff_jpeg_fdct_islow_8  ||
107 #if CONFIG_FAANDCT
108             fdsp->fdct == ff_faandct            ||
109 #endif /* CONFIG_FAANDCT */
110             fdsp->fdct == ff_jpeg_fdct_islow_10) {
111             for (i = 0; i < 64; i++) {
112                 const int j = s->idsp.idct_permutation[i];
113                 int64_t den = (int64_t) qscale2 * quant_matrix[j];
114                 /* 16 <= qscale * quant_matrix[i] <= 7905
115                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
116                  *             19952 <=              x  <= 249205026
117                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
118                  *           3444240 >= (1 << 36) / (x) >= 275 */
119
120                 qmat[qscale][i] = (int)((UINT64_C(2) << QMAT_SHIFT) / den);
121             }
122         } else if (fdsp->fdct == ff_fdct_ifast) {
123             for (i = 0; i < 64; i++) {
124                 const int j = s->idsp.idct_permutation[i];
125                 int64_t den = ff_aanscales[i] * (int64_t) qscale2 * quant_matrix[j];
126                 /* 16 <= qscale * quant_matrix[i] <= 7905
127                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
128                  *             19952 <=              x  <= 249205026
129                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
130                  *           3444240 >= (1 << 36) / (x) >= 275 */
131
132                 qmat[qscale][i] = (int)((UINT64_C(2) << (QMAT_SHIFT + 14)) / den);
133             }
134         } else {
135             for (i = 0; i < 64; i++) {
136                 const int j = s->idsp.idct_permutation[i];
137                 int64_t den = (int64_t) qscale2 * quant_matrix[j];
138                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
139                  * Assume x = qscale * quant_matrix[i]
140                  * So             16 <=              x  <= 7905
141                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
142                  * so          32768 >= (1 << 19) / (x) >= 67 */
143                 qmat[qscale][i] = (int)((UINT64_C(2) << QMAT_SHIFT) / den);
144                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
145                 //                    (qscale * quant_matrix[i]);
146                 qmat16[qscale][0][i] = (2 << QMAT_SHIFT_MMX) / den;
147
148                 if (qmat16[qscale][0][i] == 0 ||
149                     qmat16[qscale][0][i] == 128 * 256)
150                     qmat16[qscale][0][i] = 128 * 256 - 1;
151                 qmat16[qscale][1][i] =
152                     ROUNDED_DIV(bias * (1<<(16 - QUANT_BIAS_SHIFT)),
153                                 qmat16[qscale][0][i]);
154             }
155         }
156
157         for (i = intra; i < 64; i++) {
158             int64_t max = 8191;
159             if (fdsp->fdct == ff_fdct_ifast) {
160                 max = (8191LL * ff_aanscales[i]) >> 14;
161             }
162             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
163                 shift++;
164             }
165         }
166     }
167     if (shift) {
168         av_log(NULL, AV_LOG_INFO,
169                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
170                QMAT_SHIFT - shift);
171     }
172 }
173
174 static inline void update_qscale(MpegEncContext *s)
175 {
176     if (s->q_scale_type == 1 && 0) {
177         int i;
178         int bestdiff=INT_MAX;
179         int best = 1;
180
181         for (i = 0 ; i<FF_ARRAY_ELEMS(ff_mpeg2_non_linear_qscale); i++) {
182             int diff = FFABS((ff_mpeg2_non_linear_qscale[i]<<(FF_LAMBDA_SHIFT + 6)) - (int)s->lambda * 139);
183             if (ff_mpeg2_non_linear_qscale[i] < s->avctx->qmin ||
184                 (ff_mpeg2_non_linear_qscale[i] > s->avctx->qmax && !s->vbv_ignore_qmax))
185                 continue;
186             if (diff < bestdiff) {
187                 bestdiff = diff;
188                 best = i;
189             }
190         }
191         s->qscale = best;
192     } else {
193         s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
194                     (FF_LAMBDA_SHIFT + 7);
195         s->qscale = av_clip(s->qscale, s->avctx->qmin, s->vbv_ignore_qmax ? 31 : s->avctx->qmax);
196     }
197
198     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
199                  FF_LAMBDA_SHIFT;
200 }
201
202 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
203 {
204     int i;
205
206     if (matrix) {
207         put_bits(pb, 1, 1);
208         for (i = 0; i < 64; i++) {
209             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
210         }
211     } else
212         put_bits(pb, 1, 0);
213 }
214
215 /**
216  * init s->current_picture.qscale_table from s->lambda_table
217  */
218 void ff_init_qscale_tab(MpegEncContext *s)
219 {
220     int8_t * const qscale_table = s->current_picture.qscale_table;
221     int i;
222
223     for (i = 0; i < s->mb_num; i++) {
224         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
225         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
226         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
227                                                   s->avctx->qmax);
228     }
229 }
230
231 static void update_duplicate_context_after_me(MpegEncContext *dst,
232                                               MpegEncContext *src)
233 {
234 #define COPY(a) dst->a= src->a
235     COPY(pict_type);
236     COPY(current_picture);
237     COPY(f_code);
238     COPY(b_code);
239     COPY(qscale);
240     COPY(lambda);
241     COPY(lambda2);
242     COPY(picture_in_gop_number);
243     COPY(gop_picture_number);
244     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
245     COPY(progressive_frame);    // FIXME don't set in encode_header
246     COPY(partitioned_frame);    // FIXME don't set in encode_header
247 #undef COPY
248 }
249
250 /**
251  * Set the given MpegEncContext to defaults for encoding.
252  * the changed fields will not depend upon the prior state of the MpegEncContext.
253  */
254 static void mpv_encode_defaults(MpegEncContext *s)
255 {
256     int i;
257     ff_mpv_common_defaults(s);
258
259     for (i = -16; i < 16; i++) {
260         default_fcode_tab[i + MAX_MV] = 1;
261     }
262     s->me.mv_penalty = default_mv_penalty;
263     s->fcode_tab     = default_fcode_tab;
264
265     s->input_picture_number  = 0;
266     s->picture_in_gop_number = 0;
267 }
268
269 av_cold int ff_dct_encode_init(MpegEncContext *s)
270 {
271     if (ARCH_X86)
272         ff_dct_encode_init_x86(s);
273
274     if (CONFIG_H263_ENCODER)
275         ff_h263dsp_init(&s->h263dsp);
276     if (!s->dct_quantize)
277         s->dct_quantize = ff_dct_quantize_c;
278     if (!s->denoise_dct)
279         s->denoise_dct  = denoise_dct_c;
280     s->fast_dct_quantize = s->dct_quantize;
281     if (s->avctx->trellis)
282         s->dct_quantize  = dct_quantize_trellis_c;
283
284     return 0;
285 }
286
287 /* init video encoder */
288 av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
289 {
290     MpegEncContext *s = avctx->priv_data;
291     AVCPBProperties *cpb_props;
292     int i, ret, format_supported;
293
294     mpv_encode_defaults(s);
295
296     switch (avctx->codec_id) {
297     case AV_CODEC_ID_MPEG2VIDEO:
298         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
299             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
300             av_log(avctx, AV_LOG_ERROR,
301                    "only YUV420 and YUV422 are supported\n");
302             return -1;
303         }
304         break;
305     case AV_CODEC_ID_MJPEG:
306     case AV_CODEC_ID_AMV:
307         format_supported = 0;
308         /* JPEG color space */
309         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
310             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
311             avctx->pix_fmt == AV_PIX_FMT_YUVJ444P ||
312             (avctx->color_range == AVCOL_RANGE_JPEG &&
313              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
314               avctx->pix_fmt == AV_PIX_FMT_YUV422P ||
315               avctx->pix_fmt == AV_PIX_FMT_YUV444P)))
316             format_supported = 1;
317         /* MPEG color space */
318         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
319                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
320                   avctx->pix_fmt == AV_PIX_FMT_YUV422P ||
321                   avctx->pix_fmt == AV_PIX_FMT_YUV444P))
322             format_supported = 1;
323
324         if (!format_supported) {
325             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
326             return -1;
327         }
328         break;
329     default:
330         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
331             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
332             return -1;
333         }
334     }
335
336     switch (avctx->pix_fmt) {
337     case AV_PIX_FMT_YUVJ444P:
338     case AV_PIX_FMT_YUV444P:
339         s->chroma_format = CHROMA_444;
340         break;
341     case AV_PIX_FMT_YUVJ422P:
342     case AV_PIX_FMT_YUV422P:
343         s->chroma_format = CHROMA_422;
344         break;
345     case AV_PIX_FMT_YUVJ420P:
346     case AV_PIX_FMT_YUV420P:
347     default:
348         s->chroma_format = CHROMA_420;
349         break;
350     }
351
352     avctx->bits_per_raw_sample = av_clip(avctx->bits_per_raw_sample, 0, 8);
353
354 #if FF_API_PRIVATE_OPT
355 FF_DISABLE_DEPRECATION_WARNINGS
356     if (avctx->rtp_payload_size)
357         s->rtp_payload_size = avctx->rtp_payload_size;
358     if (avctx->me_penalty_compensation)
359         s->me_penalty_compensation = avctx->me_penalty_compensation;
360     if (avctx->pre_me)
361         s->me_pre = avctx->pre_me;
362 FF_ENABLE_DEPRECATION_WARNINGS
363 #endif
364
365     s->bit_rate = avctx->bit_rate;
366     s->width    = avctx->width;
367     s->height   = avctx->height;
368     if (avctx->gop_size > 600 &&
369         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
370         av_log(avctx, AV_LOG_WARNING,
371                "keyframe interval too large!, reducing it from %d to %d\n",
372                avctx->gop_size, 600);
373         avctx->gop_size = 600;
374     }
375     s->gop_size     = avctx->gop_size;
376     s->avctx        = avctx;
377     if (avctx->max_b_frames > MAX_B_FRAMES) {
378         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
379                "is %d.\n", MAX_B_FRAMES);
380         avctx->max_b_frames = MAX_B_FRAMES;
381     }
382     s->max_b_frames = avctx->max_b_frames;
383     s->codec_id     = avctx->codec->id;
384     s->strict_std_compliance = avctx->strict_std_compliance;
385     s->quarter_sample     = (avctx->flags & AV_CODEC_FLAG_QPEL) != 0;
386     s->rtp_mode           = !!s->rtp_payload_size;
387     s->intra_dc_precision = avctx->intra_dc_precision;
388
389     // workaround some differences between how applications specify dc precision
390     if (s->intra_dc_precision < 0) {
391         s->intra_dc_precision += 8;
392     } else if (s->intra_dc_precision >= 8)
393         s->intra_dc_precision -= 8;
394
395     if (s->intra_dc_precision < 0) {
396         av_log(avctx, AV_LOG_ERROR,
397                 "intra dc precision must be positive, note some applications use"
398                 " 0 and some 8 as base meaning 8bit, the value must not be smaller than that\n");
399         return AVERROR(EINVAL);
400     }
401
402     if (avctx->codec_id == AV_CODEC_ID_AMV || (avctx->active_thread_type & FF_THREAD_SLICE))
403         s->huffman = 0;
404
405     if (s->intra_dc_precision > (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO ? 3 : 0)) {
406         av_log(avctx, AV_LOG_ERROR, "intra dc precision too large\n");
407         return AVERROR(EINVAL);
408     }
409     s->user_specified_pts = AV_NOPTS_VALUE;
410
411     if (s->gop_size <= 1) {
412         s->intra_only = 1;
413         s->gop_size   = 12;
414     } else {
415         s->intra_only = 0;
416     }
417
418 #if FF_API_MOTION_EST
419 FF_DISABLE_DEPRECATION_WARNINGS
420     s->me_method = avctx->me_method;
421 FF_ENABLE_DEPRECATION_WARNINGS
422 #endif
423
424     /* Fixed QSCALE */
425     s->fixed_qscale = !!(avctx->flags & AV_CODEC_FLAG_QSCALE);
426
427 #if FF_API_MPV_OPT
428     FF_DISABLE_DEPRECATION_WARNINGS
429     if (avctx->border_masking != 0.0)
430         s->border_masking = avctx->border_masking;
431     FF_ENABLE_DEPRECATION_WARNINGS
432 #endif
433
434     s->adaptive_quant = (s->avctx->lumi_masking ||
435                          s->avctx->dark_masking ||
436                          s->avctx->temporal_cplx_masking ||
437                          s->avctx->spatial_cplx_masking  ||
438                          s->avctx->p_masking      ||
439                          s->border_masking ||
440                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
441                         !s->fixed_qscale;
442
443     s->loop_filter = !!(s->avctx->flags & AV_CODEC_FLAG_LOOP_FILTER);
444
445     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
446         switch(avctx->codec_id) {
447         case AV_CODEC_ID_MPEG1VIDEO:
448         case AV_CODEC_ID_MPEG2VIDEO:
449             avctx->rc_buffer_size = FFMAX(avctx->rc_max_rate, 15000000) * 112LL / 15000000 * 16384;
450             break;
451         case AV_CODEC_ID_MPEG4:
452         case AV_CODEC_ID_MSMPEG4V1:
453         case AV_CODEC_ID_MSMPEG4V2:
454         case AV_CODEC_ID_MSMPEG4V3:
455             if       (avctx->rc_max_rate >= 15000000) {
456                 avctx->rc_buffer_size = 320 + (avctx->rc_max_rate - 15000000LL) * (760-320) / (38400000 - 15000000);
457             } else if(avctx->rc_max_rate >=  2000000) {
458                 avctx->rc_buffer_size =  80 + (avctx->rc_max_rate -  2000000LL) * (320- 80) / (15000000 -  2000000);
459             } else if(avctx->rc_max_rate >=   384000) {
460                 avctx->rc_buffer_size =  40 + (avctx->rc_max_rate -   384000LL) * ( 80- 40) / ( 2000000 -   384000);
461             } else
462                 avctx->rc_buffer_size = 40;
463             avctx->rc_buffer_size *= 16384;
464             break;
465         }
466         if (avctx->rc_buffer_size) {
467             av_log(avctx, AV_LOG_INFO, "Automatically choosing VBV buffer size of %d kbyte\n", avctx->rc_buffer_size/8192);
468         }
469     }
470
471     if ((!avctx->rc_max_rate) != (!avctx->rc_buffer_size)) {
472         av_log(avctx, AV_LOG_ERROR, "Either both buffer size and max rate or neither must be specified\n");
473         return -1;
474     }
475
476     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
477         av_log(avctx, AV_LOG_INFO,
478                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
479     }
480
481     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
482         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
483         return -1;
484     }
485
486     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
487         av_log(avctx, AV_LOG_ERROR, "bitrate above max bitrate\n");
488         return -1;
489     }
490
491     if (avctx->rc_max_rate &&
492         avctx->rc_max_rate == avctx->bit_rate &&
493         avctx->rc_max_rate != avctx->rc_min_rate) {
494         av_log(avctx, AV_LOG_INFO,
495                "impossible bitrate constraints, this will fail\n");
496     }
497
498     if (avctx->rc_buffer_size &&
499         avctx->bit_rate * (int64_t)avctx->time_base.num >
500             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
501         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
502         return -1;
503     }
504
505     if (!s->fixed_qscale &&
506         avctx->bit_rate * av_q2d(avctx->time_base) >
507             avctx->bit_rate_tolerance) {
508         av_log(avctx, AV_LOG_WARNING,
509                "bitrate tolerance %d too small for bitrate %"PRId64", overriding\n", avctx->bit_rate_tolerance, (int64_t)avctx->bit_rate);
510         avctx->bit_rate_tolerance = 5 * avctx->bit_rate * av_q2d(avctx->time_base);
511     }
512
513     if (s->avctx->rc_max_rate &&
514         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
515         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
516          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
517         90000LL * (avctx->rc_buffer_size - 1) >
518             s->avctx->rc_max_rate * 0xFFFFLL) {
519         av_log(avctx, AV_LOG_INFO,
520                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
521                "specified vbv buffer is too large for the given bitrate!\n");
522     }
523
524     if ((s->avctx->flags & AV_CODEC_FLAG_4MV) && s->codec_id != AV_CODEC_ID_MPEG4 &&
525         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
526         s->codec_id != AV_CODEC_ID_FLV1) {
527         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
528         return -1;
529     }
530
531     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
532         av_log(avctx, AV_LOG_ERROR,
533                "OBMC is only supported with simple mb decision\n");
534         return -1;
535     }
536
537     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
538         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
539         return -1;
540     }
541
542     if (s->max_b_frames                    &&
543         s->codec_id != AV_CODEC_ID_MPEG4      &&
544         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
545         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
546         av_log(avctx, AV_LOG_ERROR, "B-frames not supported by codec\n");
547         return -1;
548     }
549     if (s->max_b_frames < 0) {
550         av_log(avctx, AV_LOG_ERROR,
551                "max b frames must be 0 or positive for mpegvideo based encoders\n");
552         return -1;
553     }
554
555     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
556          s->codec_id == AV_CODEC_ID_H263  ||
557          s->codec_id == AV_CODEC_ID_H263P) &&
558         (avctx->sample_aspect_ratio.num > 255 ||
559          avctx->sample_aspect_ratio.den > 255)) {
560         av_log(avctx, AV_LOG_WARNING,
561                "Invalid pixel aspect ratio %i/%i, limit is 255/255 reducing\n",
562                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
563         av_reduce(&avctx->sample_aspect_ratio.num, &avctx->sample_aspect_ratio.den,
564                    avctx->sample_aspect_ratio.num,  avctx->sample_aspect_ratio.den, 255);
565     }
566
567     if ((s->codec_id == AV_CODEC_ID_H263  ||
568          s->codec_id == AV_CODEC_ID_H263P) &&
569         (avctx->width  > 2048 ||
570          avctx->height > 1152 )) {
571         av_log(avctx, AV_LOG_ERROR, "H.263 does not support resolutions above 2048x1152\n");
572         return -1;
573     }
574     if ((s->codec_id == AV_CODEC_ID_H263  ||
575          s->codec_id == AV_CODEC_ID_H263P) &&
576         ((avctx->width &3) ||
577          (avctx->height&3) )) {
578         av_log(avctx, AV_LOG_ERROR, "w/h must be a multiple of 4\n");
579         return -1;
580     }
581
582     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO &&
583         (avctx->width  > 4095 ||
584          avctx->height > 4095 )) {
585         av_log(avctx, AV_LOG_ERROR, "MPEG-1 does not support resolutions above 4095x4095\n");
586         return -1;
587     }
588
589     if (s->codec_id == AV_CODEC_ID_MPEG2VIDEO &&
590         (avctx->width  > 16383 ||
591          avctx->height > 16383 )) {
592         av_log(avctx, AV_LOG_ERROR, "MPEG-2 does not support resolutions above 16383x16383\n");
593         return -1;
594     }
595
596     if (s->codec_id == AV_CODEC_ID_RV10 &&
597         (avctx->width &15 ||
598          avctx->height&15 )) {
599         av_log(avctx, AV_LOG_ERROR, "width and height must be a multiple of 16\n");
600         return AVERROR(EINVAL);
601     }
602
603     if (s->codec_id == AV_CODEC_ID_RV20 &&
604         (avctx->width &3 ||
605          avctx->height&3 )) {
606         av_log(avctx, AV_LOG_ERROR, "width and height must be a multiple of 4\n");
607         return AVERROR(EINVAL);
608     }
609
610     if ((s->codec_id == AV_CODEC_ID_WMV1 ||
611          s->codec_id == AV_CODEC_ID_WMV2) &&
612          avctx->width & 1) {
613          av_log(avctx, AV_LOG_ERROR, "width must be multiple of 2\n");
614          return -1;
615     }
616
617     if ((s->avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT | AV_CODEC_FLAG_INTERLACED_ME)) &&
618         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
619         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
620         return -1;
621     }
622
623 #if FF_API_PRIVATE_OPT
624     FF_DISABLE_DEPRECATION_WARNINGS
625     if (avctx->mpeg_quant)
626         s->mpeg_quant = avctx->mpeg_quant;
627     FF_ENABLE_DEPRECATION_WARNINGS
628 #endif
629
630     // FIXME mpeg2 uses that too
631     if (s->mpeg_quant && (   s->codec_id != AV_CODEC_ID_MPEG4
632                           && s->codec_id != AV_CODEC_ID_MPEG2VIDEO)) {
633         av_log(avctx, AV_LOG_ERROR,
634                "mpeg2 style quantization not supported by codec\n");
635         return -1;
636     }
637
638     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
639         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
640         return -1;
641     }
642
643     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
644         s->avctx->mb_decision != FF_MB_DECISION_RD) {
645         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
646         return -1;
647     }
648
649     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
650             (s->codec_id == AV_CODEC_ID_AMV ||
651              s->codec_id == AV_CODEC_ID_MJPEG)) {
652         // Used to produce garbage with MJPEG.
653         av_log(avctx, AV_LOG_ERROR,
654                "QP RD is no longer compatible with MJPEG or AMV\n");
655         return -1;
656     }
657
658 #if FF_API_PRIVATE_OPT
659 FF_DISABLE_DEPRECATION_WARNINGS
660     if (avctx->scenechange_threshold)
661         s->scenechange_threshold = avctx->scenechange_threshold;
662 FF_ENABLE_DEPRECATION_WARNINGS
663 #endif
664
665     if (s->scenechange_threshold < 1000000000 &&
666         (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)) {
667         av_log(avctx, AV_LOG_ERROR,
668                "closed gop with scene change detection are not supported yet, "
669                "set threshold to 1000000000\n");
670         return -1;
671     }
672
673     if (s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY) {
674         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
675             s->strict_std_compliance >= FF_COMPLIANCE_NORMAL) {
676             av_log(avctx, AV_LOG_ERROR,
677                    "low delay forcing is only available for mpeg2, "
678                    "set strict_std_compliance to 'unofficial' or lower in order to allow it\n");
679             return -1;
680         }
681         if (s->max_b_frames != 0) {
682             av_log(avctx, AV_LOG_ERROR,
683                    "B-frames cannot be used with low delay\n");
684             return -1;
685         }
686     }
687
688     if (s->q_scale_type == 1) {
689         if (avctx->qmax > 28) {
690             av_log(avctx, AV_LOG_ERROR,
691                    "non linear quant only supports qmax <= 28 currently\n");
692             return -1;
693         }
694     }
695
696     if (avctx->slices > 1 &&
697         (avctx->codec_id == AV_CODEC_ID_FLV1 || avctx->codec_id == AV_CODEC_ID_H261)) {
698         av_log(avctx, AV_LOG_ERROR, "Multiple slices are not supported by this codec\n");
699         return AVERROR(EINVAL);
700     }
701
702     if (s->avctx->thread_count > 1         &&
703         s->codec_id != AV_CODEC_ID_MPEG4      &&
704         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
705         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
706         s->codec_id != AV_CODEC_ID_MJPEG      &&
707         (s->codec_id != AV_CODEC_ID_H263P)) {
708         av_log(avctx, AV_LOG_ERROR,
709                "multi threaded encoding not supported by codec\n");
710         return -1;
711     }
712
713     if (s->avctx->thread_count < 1) {
714         av_log(avctx, AV_LOG_ERROR,
715                "automatic thread number detection not supported by codec, "
716                "patch welcome\n");
717         return -1;
718     }
719
720     if (!avctx->time_base.den || !avctx->time_base.num) {
721         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
722         return -1;
723     }
724
725 #if FF_API_PRIVATE_OPT
726 FF_DISABLE_DEPRECATION_WARNINGS
727     if (avctx->b_frame_strategy)
728         s->b_frame_strategy = avctx->b_frame_strategy;
729     if (avctx->b_sensitivity != 40)
730         s->b_sensitivity = avctx->b_sensitivity;
731 FF_ENABLE_DEPRECATION_WARNINGS
732 #endif
733
734     if (s->b_frame_strategy && (avctx->flags & AV_CODEC_FLAG_PASS2)) {
735         av_log(avctx, AV_LOG_INFO,
736                "notice: b_frame_strategy only affects the first pass\n");
737         s->b_frame_strategy = 0;
738     }
739
740     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
741     if (i > 1) {
742         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
743         avctx->time_base.den /= i;
744         avctx->time_base.num /= i;
745         //return -1;
746     }
747
748     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG || s->codec_id==AV_CODEC_ID_AMV) {
749         // (a + x * 3 / 8) / x
750         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
751         s->inter_quant_bias = 0;
752     } else {
753         s->intra_quant_bias = 0;
754         // (a - x / 4) / x
755         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
756     }
757
758     if (avctx->qmin > avctx->qmax || avctx->qmin <= 0) {
759         av_log(avctx, AV_LOG_ERROR, "qmin and or qmax are invalid, they must be 0 < min <= max\n");
760         return AVERROR(EINVAL);
761     }
762
763 #if FF_API_QUANT_BIAS
764 FF_DISABLE_DEPRECATION_WARNINGS
765     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
766         s->intra_quant_bias = avctx->intra_quant_bias;
767     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
768         s->inter_quant_bias = avctx->inter_quant_bias;
769 FF_ENABLE_DEPRECATION_WARNINGS
770 #endif
771
772     av_log(avctx, AV_LOG_DEBUG, "intra_quant_bias = %d inter_quant_bias = %d\n",s->intra_quant_bias,s->inter_quant_bias);
773
774     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
775         s->avctx->time_base.den > (1 << 16) - 1) {
776         av_log(avctx, AV_LOG_ERROR,
777                "timebase %d/%d not supported by MPEG 4 standard, "
778                "the maximum admitted value for the timebase denominator "
779                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
780                (1 << 16) - 1);
781         return -1;
782     }
783     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
784
785     switch (avctx->codec->id) {
786     case AV_CODEC_ID_MPEG1VIDEO:
787         s->out_format = FMT_MPEG1;
788         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
789         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
790         break;
791     case AV_CODEC_ID_MPEG2VIDEO:
792         s->out_format = FMT_MPEG1;
793         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
794         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
795         s->rtp_mode   = 1;
796         break;
797     case AV_CODEC_ID_MJPEG:
798     case AV_CODEC_ID_AMV:
799         s->out_format = FMT_MJPEG;
800         s->intra_only = 1; /* force intra only for jpeg */
801         if (!CONFIG_MJPEG_ENCODER ||
802             ff_mjpeg_encode_init(s) < 0)
803             return -1;
804         avctx->delay = 0;
805         s->low_delay = 1;
806         break;
807     case AV_CODEC_ID_H261:
808         if (!CONFIG_H261_ENCODER)
809             return -1;
810         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
811             av_log(avctx, AV_LOG_ERROR,
812                    "The specified picture size of %dx%d is not valid for the "
813                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
814                     s->width, s->height);
815             return -1;
816         }
817         s->out_format = FMT_H261;
818         avctx->delay  = 0;
819         s->low_delay  = 1;
820         s->rtp_mode   = 0; /* Sliced encoding not supported */
821         break;
822     case AV_CODEC_ID_H263:
823         if (!CONFIG_H263_ENCODER)
824             return -1;
825         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
826                              s->width, s->height) == 8) {
827             av_log(avctx, AV_LOG_ERROR,
828                    "The specified picture size of %dx%d is not valid for "
829                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
830                    "352x288, 704x576, and 1408x1152. "
831                    "Try H.263+.\n", s->width, s->height);
832             return -1;
833         }
834         s->out_format = FMT_H263;
835         avctx->delay  = 0;
836         s->low_delay  = 1;
837         break;
838     case AV_CODEC_ID_H263P:
839         s->out_format = FMT_H263;
840         s->h263_plus  = 1;
841         /* Fx */
842         s->h263_aic        = (avctx->flags & AV_CODEC_FLAG_AC_PRED) ? 1 : 0;
843         s->modified_quant  = s->h263_aic;
844         s->loop_filter     = (avctx->flags & AV_CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
845         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
846
847         /* /Fx */
848         /* These are just to be sure */
849         avctx->delay = 0;
850         s->low_delay = 1;
851         break;
852     case AV_CODEC_ID_FLV1:
853         s->out_format      = FMT_H263;
854         s->h263_flv        = 2; /* format = 1; 11-bit codes */
855         s->unrestricted_mv = 1;
856         s->rtp_mode  = 0; /* don't allow GOB */
857         avctx->delay = 0;
858         s->low_delay = 1;
859         break;
860     case AV_CODEC_ID_RV10:
861         s->out_format = FMT_H263;
862         avctx->delay  = 0;
863         s->low_delay  = 1;
864         break;
865     case AV_CODEC_ID_RV20:
866         s->out_format      = FMT_H263;
867         avctx->delay       = 0;
868         s->low_delay       = 1;
869         s->modified_quant  = 1;
870         s->h263_aic        = 1;
871         s->h263_plus       = 1;
872         s->loop_filter     = 1;
873         s->unrestricted_mv = 0;
874         break;
875     case AV_CODEC_ID_MPEG4:
876         s->out_format      = FMT_H263;
877         s->h263_pred       = 1;
878         s->unrestricted_mv = 1;
879         s->low_delay       = s->max_b_frames ? 0 : 1;
880         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
881         break;
882     case AV_CODEC_ID_MSMPEG4V2:
883         s->out_format      = FMT_H263;
884         s->h263_pred       = 1;
885         s->unrestricted_mv = 1;
886         s->msmpeg4_version = 2;
887         avctx->delay       = 0;
888         s->low_delay       = 1;
889         break;
890     case AV_CODEC_ID_MSMPEG4V3:
891         s->out_format        = FMT_H263;
892         s->h263_pred         = 1;
893         s->unrestricted_mv   = 1;
894         s->msmpeg4_version   = 3;
895         s->flipflop_rounding = 1;
896         avctx->delay         = 0;
897         s->low_delay         = 1;
898         break;
899     case AV_CODEC_ID_WMV1:
900         s->out_format        = FMT_H263;
901         s->h263_pred         = 1;
902         s->unrestricted_mv   = 1;
903         s->msmpeg4_version   = 4;
904         s->flipflop_rounding = 1;
905         avctx->delay         = 0;
906         s->low_delay         = 1;
907         break;
908     case AV_CODEC_ID_WMV2:
909         s->out_format        = FMT_H263;
910         s->h263_pred         = 1;
911         s->unrestricted_mv   = 1;
912         s->msmpeg4_version   = 5;
913         s->flipflop_rounding = 1;
914         avctx->delay         = 0;
915         s->low_delay         = 1;
916         break;
917     default:
918         return -1;
919     }
920
921 #if FF_API_PRIVATE_OPT
922     FF_DISABLE_DEPRECATION_WARNINGS
923     if (avctx->noise_reduction)
924         s->noise_reduction = avctx->noise_reduction;
925     FF_ENABLE_DEPRECATION_WARNINGS
926 #endif
927
928     avctx->has_b_frames = !s->low_delay;
929
930     s->encoding = 1;
931
932     s->progressive_frame    =
933     s->progressive_sequence = !(avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT |
934                                                 AV_CODEC_FLAG_INTERLACED_ME) ||
935                                 s->alternate_scan);
936
937     /* init */
938     ff_mpv_idct_init(s);
939     if (ff_mpv_common_init(s) < 0)
940         return -1;
941
942     ff_fdctdsp_init(&s->fdsp, avctx);
943     ff_me_cmp_init(&s->mecc, avctx);
944     ff_mpegvideoencdsp_init(&s->mpvencdsp, avctx);
945     ff_pixblockdsp_init(&s->pdsp, avctx);
946     ff_qpeldsp_init(&s->qdsp);
947
948     if (s->msmpeg4_version) {
949         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
950                           2 * 2 * (MAX_LEVEL + 1) *
951                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
952     }
953     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
954
955     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
956     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_chroma_intra_matrix, 64 * 32 * sizeof(int), fail);
957     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
958     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
959     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_chroma_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
960     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
961     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
962                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
963     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
964                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
965
966
967     if (s->noise_reduction) {
968         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
969                           2 * 64 * sizeof(uint16_t), fail);
970     }
971
972     ff_dct_encode_init(s);
973
974     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
975         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
976
977     if (s->slice_context_count > 1) {
978         s->rtp_mode = 1;
979
980         if (avctx->codec_id == AV_CODEC_ID_H263P)
981             s->h263_slice_structured = 1;
982     }
983
984     s->quant_precision = 5;
985
986 #if FF_API_PRIVATE_OPT
987 FF_DISABLE_DEPRECATION_WARNINGS
988     if (avctx->frame_skip_threshold)
989         s->frame_skip_threshold = avctx->frame_skip_threshold;
990     if (avctx->frame_skip_factor)
991         s->frame_skip_factor = avctx->frame_skip_factor;
992     if (avctx->frame_skip_exp)
993         s->frame_skip_exp = avctx->frame_skip_exp;
994     if (avctx->frame_skip_cmp != FF_CMP_DCTMAX)
995         s->frame_skip_cmp = avctx->frame_skip_cmp;
996 FF_ENABLE_DEPRECATION_WARNINGS
997 #endif
998
999     ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      s->avctx->ildct_cmp);
1000     ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp);
1001
1002     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
1003         ff_h261_encode_init(s);
1004     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
1005         ff_h263_encode_init(s);
1006     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
1007         if ((ret = ff_msmpeg4_encode_init(s)) < 0)
1008             return ret;
1009     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
1010         && s->out_format == FMT_MPEG1)
1011         ff_mpeg1_encode_init(s);
1012
1013     /* init q matrix */
1014     for (i = 0; i < 64; i++) {
1015         int j = s->idsp.idct_permutation[i];
1016         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
1017             s->mpeg_quant) {
1018             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
1019             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
1020         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1021             s->intra_matrix[j] =
1022             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
1023         } else {
1024             /* MPEG-1/2 */
1025             s->chroma_intra_matrix[j] =
1026             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
1027             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
1028         }
1029         if (s->avctx->intra_matrix)
1030             s->intra_matrix[j] = s->avctx->intra_matrix[i];
1031         if (s->avctx->inter_matrix)
1032             s->inter_matrix[j] = s->avctx->inter_matrix[i];
1033     }
1034
1035     /* precompute matrix */
1036     /* for mjpeg, we do include qscale in the matrix */
1037     if (s->out_format != FMT_MJPEG) {
1038         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
1039                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
1040                           31, 1);
1041         ff_convert_matrix(s, s->q_inter_matrix, s->q_inter_matrix16,
1042                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
1043                           31, 0);
1044     }
1045
1046 #if FF_API_RC_STRATEGY
1047 FF_DISABLE_DEPRECATION_WARNINGS
1048     if (!s->rc_strategy)
1049         s->rc_strategy = s->avctx->rc_strategy;
1050 FF_ENABLE_DEPRECATION_WARNINGS
1051 #endif
1052
1053     if (ff_rate_control_init(s) < 0)
1054         return -1;
1055
1056 #if FF_API_RC_STRATEGY
1057     av_assert0(MPV_RC_STRATEGY_XVID == FF_RC_STRATEGY_XVID);
1058 #endif
1059
1060     if ((s->avctx->flags & AV_CODEC_FLAG_PASS2) && s->rc_strategy == MPV_RC_STRATEGY_XVID) {
1061 #if CONFIG_LIBXVID
1062         ret = ff_xvid_rate_control_init(s);
1063 #else
1064         ret = AVERROR(ENOSYS);
1065         av_log(s->avctx, AV_LOG_ERROR,
1066                "Xvid ratecontrol requires libavcodec compiled with Xvid support.\n");
1067 #endif
1068         if (ret < 0)
1069             return ret;
1070     }
1071
1072 #if FF_API_ERROR_RATE
1073     FF_DISABLE_DEPRECATION_WARNINGS
1074     if (avctx->error_rate)
1075         s->error_rate = avctx->error_rate;
1076     FF_ENABLE_DEPRECATION_WARNINGS;
1077 #endif
1078
1079 #if FF_API_NORMALIZE_AQP
1080     FF_DISABLE_DEPRECATION_WARNINGS
1081     if (avctx->flags & CODEC_FLAG_NORMALIZE_AQP)
1082         s->mpv_flags |= FF_MPV_FLAG_NAQ;
1083     FF_ENABLE_DEPRECATION_WARNINGS;
1084 #endif
1085
1086 #if FF_API_MV0
1087     FF_DISABLE_DEPRECATION_WARNINGS
1088     if (avctx->flags & CODEC_FLAG_MV0)
1089         s->mpv_flags |= FF_MPV_FLAG_MV0;
1090     FF_ENABLE_DEPRECATION_WARNINGS
1091 #endif
1092
1093 #if FF_API_MPV_OPT
1094     FF_DISABLE_DEPRECATION_WARNINGS
1095     if (avctx->rc_qsquish != 0.0)
1096         s->rc_qsquish = avctx->rc_qsquish;
1097     if (avctx->rc_qmod_amp != 0.0)
1098         s->rc_qmod_amp = avctx->rc_qmod_amp;
1099     if (avctx->rc_qmod_freq)
1100         s->rc_qmod_freq = avctx->rc_qmod_freq;
1101     if (avctx->rc_buffer_aggressivity != 1.0)
1102         s->rc_buffer_aggressivity = avctx->rc_buffer_aggressivity;
1103     if (avctx->rc_initial_cplx != 0.0)
1104         s->rc_initial_cplx = avctx->rc_initial_cplx;
1105     if (avctx->lmin)
1106         s->lmin = avctx->lmin;
1107     if (avctx->lmax)
1108         s->lmax = avctx->lmax;
1109
1110     if (avctx->rc_eq) {
1111         av_freep(&s->rc_eq);
1112         s->rc_eq = av_strdup(avctx->rc_eq);
1113         if (!s->rc_eq)
1114             return AVERROR(ENOMEM);
1115     }
1116     FF_ENABLE_DEPRECATION_WARNINGS
1117 #endif
1118
1119 #if FF_API_PRIVATE_OPT
1120     FF_DISABLE_DEPRECATION_WARNINGS
1121     if (avctx->brd_scale)
1122         s->brd_scale = avctx->brd_scale;
1123
1124     if (avctx->prediction_method)
1125         s->pred = avctx->prediction_method + 1;
1126     FF_ENABLE_DEPRECATION_WARNINGS
1127 #endif
1128
1129     if (s->b_frame_strategy == 2) {
1130         for (i = 0; i < s->max_b_frames + 2; i++) {
1131             s->tmp_frames[i] = av_frame_alloc();
1132             if (!s->tmp_frames[i])
1133                 return AVERROR(ENOMEM);
1134
1135             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
1136             s->tmp_frames[i]->width  = s->width  >> s->brd_scale;
1137             s->tmp_frames[i]->height = s->height >> s->brd_scale;
1138
1139             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
1140             if (ret < 0)
1141                 return ret;
1142         }
1143     }
1144
1145     cpb_props = ff_add_cpb_side_data(avctx);
1146     if (!cpb_props)
1147         return AVERROR(ENOMEM);
1148     cpb_props->max_bitrate = avctx->rc_max_rate;
1149     cpb_props->min_bitrate = avctx->rc_min_rate;
1150     cpb_props->avg_bitrate = avctx->bit_rate;
1151     cpb_props->buffer_size = avctx->rc_buffer_size;
1152
1153     return 0;
1154 fail:
1155     ff_mpv_encode_end(avctx);
1156     return AVERROR_UNKNOWN;
1157 }
1158
1159 av_cold int ff_mpv_encode_end(AVCodecContext *avctx)
1160 {
1161     MpegEncContext *s = avctx->priv_data;
1162     int i;
1163
1164     ff_rate_control_uninit(s);
1165 #if CONFIG_LIBXVID
1166     if ((avctx->flags & AV_CODEC_FLAG_PASS2) && s->rc_strategy == MPV_RC_STRATEGY_XVID)
1167         ff_xvid_rate_control_uninit(s);
1168 #endif
1169
1170     ff_mpv_common_end(s);
1171     if (CONFIG_MJPEG_ENCODER &&
1172         s->out_format == FMT_MJPEG)
1173         ff_mjpeg_encode_close(s);
1174
1175     av_freep(&avctx->extradata);
1176
1177     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
1178         av_frame_free(&s->tmp_frames[i]);
1179
1180     ff_free_picture_tables(&s->new_picture);
1181     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1182
1183     av_freep(&s->avctx->stats_out);
1184     av_freep(&s->ac_stats);
1185
1186     if(s->q_chroma_intra_matrix   != s->q_intra_matrix  ) av_freep(&s->q_chroma_intra_matrix);
1187     if(s->q_chroma_intra_matrix16 != s->q_intra_matrix16) av_freep(&s->q_chroma_intra_matrix16);
1188     s->q_chroma_intra_matrix=   NULL;
1189     s->q_chroma_intra_matrix16= NULL;
1190     av_freep(&s->q_intra_matrix);
1191     av_freep(&s->q_inter_matrix);
1192     av_freep(&s->q_intra_matrix16);
1193     av_freep(&s->q_inter_matrix16);
1194     av_freep(&s->input_picture);
1195     av_freep(&s->reordered_input_picture);
1196     av_freep(&s->dct_offset);
1197
1198     return 0;
1199 }
1200
1201 static int get_sae(uint8_t *src, int ref, int stride)
1202 {
1203     int x,y;
1204     int acc = 0;
1205
1206     for (y = 0; y < 16; y++) {
1207         for (x = 0; x < 16; x++) {
1208             acc += FFABS(src[x + y * stride] - ref);
1209         }
1210     }
1211
1212     return acc;
1213 }
1214
1215 static int get_intra_count(MpegEncContext *s, uint8_t *src,
1216                            uint8_t *ref, int stride)
1217 {
1218     int x, y, w, h;
1219     int acc = 0;
1220
1221     w = s->width  & ~15;
1222     h = s->height & ~15;
1223
1224     for (y = 0; y < h; y += 16) {
1225         for (x = 0; x < w; x += 16) {
1226             int offset = x + y * stride;
1227             int sad  = s->mecc.sad[0](NULL, src + offset, ref + offset,
1228                                       stride, 16);
1229             int mean = (s->mpvencdsp.pix_sum(src + offset, stride) + 128) >> 8;
1230             int sae  = get_sae(src + offset, mean, stride);
1231
1232             acc += sae + 500 < sad;
1233         }
1234     }
1235     return acc;
1236 }
1237
1238 static int alloc_picture(MpegEncContext *s, Picture *pic, int shared)
1239 {
1240     return ff_alloc_picture(s->avctx, pic, &s->me, &s->sc, shared, 1,
1241                             s->chroma_x_shift, s->chroma_y_shift, s->out_format,
1242                             s->mb_stride, s->mb_width, s->mb_height, s->b8_stride,
1243                             &s->linesize, &s->uvlinesize);
1244 }
1245
1246 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
1247 {
1248     Picture *pic = NULL;
1249     int64_t pts;
1250     int i, display_picture_number = 0, ret;
1251     int encoding_delay = s->max_b_frames ? s->max_b_frames
1252                                          : (s->low_delay ? 0 : 1);
1253     int flush_offset = 1;
1254     int direct = 1;
1255
1256     if (pic_arg) {
1257         pts = pic_arg->pts;
1258         display_picture_number = s->input_picture_number++;
1259
1260         if (pts != AV_NOPTS_VALUE) {
1261             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1262                 int64_t last = s->user_specified_pts;
1263
1264                 if (pts <= last) {
1265                     av_log(s->avctx, AV_LOG_ERROR,
1266                            "Invalid pts (%"PRId64") <= last (%"PRId64")\n",
1267                            pts, last);
1268                     return AVERROR(EINVAL);
1269                 }
1270
1271                 if (!s->low_delay && display_picture_number == 1)
1272                     s->dts_delta = pts - last;
1273             }
1274             s->user_specified_pts = pts;
1275         } else {
1276             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1277                 s->user_specified_pts =
1278                 pts = s->user_specified_pts + 1;
1279                 av_log(s->avctx, AV_LOG_INFO,
1280                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
1281                        pts);
1282             } else {
1283                 pts = display_picture_number;
1284             }
1285         }
1286
1287         if (!pic_arg->buf[0] ||
1288             pic_arg->linesize[0] != s->linesize ||
1289             pic_arg->linesize[1] != s->uvlinesize ||
1290             pic_arg->linesize[2] != s->uvlinesize)
1291             direct = 0;
1292         if ((s->width & 15) || (s->height & 15))
1293             direct = 0;
1294         if (((intptr_t)(pic_arg->data[0])) & (STRIDE_ALIGN-1))
1295             direct = 0;
1296         if (s->linesize & (STRIDE_ALIGN-1))
1297             direct = 0;
1298
1299         ff_dlog(s->avctx, "%d %d %"PTRDIFF_SPECIFIER" %"PTRDIFF_SPECIFIER"\n", pic_arg->linesize[0],
1300                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
1301
1302         i = ff_find_unused_picture(s->avctx, s->picture, direct);
1303         if (i < 0)
1304             return i;
1305
1306         pic = &s->picture[i];
1307         pic->reference = 3;
1308
1309         if (direct) {
1310             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
1311                 return ret;
1312         }
1313         ret = alloc_picture(s, pic, direct);
1314         if (ret < 0)
1315             return ret;
1316
1317         if (!direct) {
1318             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1319                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1320                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1321                 // empty
1322             } else {
1323                 int h_chroma_shift, v_chroma_shift;
1324                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
1325                                                  &h_chroma_shift,
1326                                                  &v_chroma_shift);
1327
1328                 for (i = 0; i < 3; i++) {
1329                     int src_stride = pic_arg->linesize[i];
1330                     int dst_stride = i ? s->uvlinesize : s->linesize;
1331                     int h_shift = i ? h_chroma_shift : 0;
1332                     int v_shift = i ? v_chroma_shift : 0;
1333                     int w = s->width  >> h_shift;
1334                     int h = s->height >> v_shift;
1335                     uint8_t *src = pic_arg->data[i];
1336                     uint8_t *dst = pic->f->data[i];
1337                     int vpad = 16;
1338
1339                     if (   s->codec_id == AV_CODEC_ID_MPEG2VIDEO
1340                         && !s->progressive_sequence
1341                         && FFALIGN(s->height, 32) - s->height > 16)
1342                         vpad = 32;
1343
1344                     if (!s->avctx->rc_buffer_size)
1345                         dst += INPLACE_OFFSET;
1346
1347                     if (src_stride == dst_stride)
1348                         memcpy(dst, src, src_stride * h);
1349                     else {
1350                         int h2 = h;
1351                         uint8_t *dst2 = dst;
1352                         while (h2--) {
1353                             memcpy(dst2, src, w);
1354                             dst2 += dst_stride;
1355                             src += src_stride;
1356                         }
1357                     }
1358                     if ((s->width & 15) || (s->height & (vpad-1))) {
1359                         s->mpvencdsp.draw_edges(dst, dst_stride,
1360                                                 w, h,
1361                                                 16 >> h_shift,
1362                                                 vpad >> v_shift,
1363                                                 EDGE_BOTTOM);
1364                     }
1365                 }
1366                 emms_c();
1367             }
1368         }
1369         ret = av_frame_copy_props(pic->f, pic_arg);
1370         if (ret < 0)
1371             return ret;
1372
1373         pic->f->display_picture_number = display_picture_number;
1374         pic->f->pts = pts; // we set this here to avoid modifying pic_arg
1375     } else {
1376         /* Flushing: When we have not received enough input frames,
1377          * ensure s->input_picture[0] contains the first picture */
1378         for (flush_offset = 0; flush_offset < encoding_delay + 1; flush_offset++)
1379             if (s->input_picture[flush_offset])
1380                 break;
1381
1382         if (flush_offset <= 1)
1383             flush_offset = 1;
1384         else
1385             encoding_delay = encoding_delay - flush_offset + 1;
1386     }
1387
1388     /* shift buffer entries */
1389     for (i = flush_offset; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1390         s->input_picture[i - flush_offset] = s->input_picture[i];
1391
1392     s->input_picture[encoding_delay] = (Picture*) pic;
1393
1394     return 0;
1395 }
1396
1397 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1398 {
1399     int x, y, plane;
1400     int score = 0;
1401     int64_t score64 = 0;
1402
1403     for (plane = 0; plane < 3; plane++) {
1404         const int stride = p->f->linesize[plane];
1405         const int bw = plane ? 1 : 2;
1406         for (y = 0; y < s->mb_height * bw; y++) {
1407             for (x = 0; x < s->mb_width * bw; x++) {
1408                 int off = p->shared ? 0 : 16;
1409                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1410                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1411                 int v = s->mecc.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1412
1413                 switch (FFABS(s->frame_skip_exp)) {
1414                 case 0: score    =  FFMAX(score, v);          break;
1415                 case 1: score   += FFABS(v);                  break;
1416                 case 2: score64 += v * (int64_t)v;                       break;
1417                 case 3: score64 += FFABS(v * (int64_t)v * v);            break;
1418                 case 4: score64 += (v * (int64_t)v) * (v * (int64_t)v);  break;
1419                 }
1420             }
1421         }
1422     }
1423     emms_c();
1424
1425     if (score)
1426         score64 = score;
1427     if (s->frame_skip_exp < 0)
1428         score64 = pow(score64 / (double)(s->mb_width * s->mb_height),
1429                       -1.0/s->frame_skip_exp);
1430
1431     if (score64 < s->frame_skip_threshold)
1432         return 1;
1433     if (score64 < ((s->frame_skip_factor * (int64_t) s->lambda) >> 8))
1434         return 1;
1435     return 0;
1436 }
1437
1438 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1439 {
1440     AVPacket pkt = { 0 };
1441     int ret;
1442     int size = 0;
1443
1444     av_init_packet(&pkt);
1445
1446     ret = avcodec_send_frame(c, frame);
1447     if (ret < 0)
1448         return ret;
1449
1450     do {
1451         ret = avcodec_receive_packet(c, &pkt);
1452         if (ret >= 0) {
1453             size += pkt.size;
1454             av_packet_unref(&pkt);
1455         } else if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
1456             return ret;
1457     } while (ret >= 0);
1458
1459     return size;
1460 }
1461
1462 static int estimate_best_b_count(MpegEncContext *s)
1463 {
1464     const AVCodec *codec = avcodec_find_encoder(s->avctx->codec_id);
1465     const int scale = s->brd_scale;
1466     int width  = s->width  >> scale;
1467     int height = s->height >> scale;
1468     int i, j, out_size, p_lambda, b_lambda, lambda2;
1469     int64_t best_rd  = INT64_MAX;
1470     int best_b_count = -1;
1471     int ret = 0;
1472
1473     av_assert0(scale >= 0 && scale <= 3);
1474
1475     //emms_c();
1476     //s->next_picture_ptr->quality;
1477     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1478     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1479     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1480     if (!b_lambda) // FIXME we should do this somewhere else
1481         b_lambda = p_lambda;
1482     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1483                FF_LAMBDA_SHIFT;
1484
1485     for (i = 0; i < s->max_b_frames + 2; i++) {
1486         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1487                                                 s->next_picture_ptr;
1488         uint8_t *data[4];
1489
1490         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1491             pre_input = *pre_input_ptr;
1492             memcpy(data, pre_input_ptr->f->data, sizeof(data));
1493
1494             if (!pre_input.shared && i) {
1495                 data[0] += INPLACE_OFFSET;
1496                 data[1] += INPLACE_OFFSET;
1497                 data[2] += INPLACE_OFFSET;
1498             }
1499
1500             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[0],
1501                                        s->tmp_frames[i]->linesize[0],
1502                                        data[0],
1503                                        pre_input.f->linesize[0],
1504                                        width, height);
1505             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[1],
1506                                        s->tmp_frames[i]->linesize[1],
1507                                        data[1],
1508                                        pre_input.f->linesize[1],
1509                                        width >> 1, height >> 1);
1510             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[2],
1511                                        s->tmp_frames[i]->linesize[2],
1512                                        data[2],
1513                                        pre_input.f->linesize[2],
1514                                        width >> 1, height >> 1);
1515         }
1516     }
1517
1518     for (j = 0; j < s->max_b_frames + 1; j++) {
1519         AVCodecContext *c;
1520         int64_t rd = 0;
1521
1522         if (!s->input_picture[j])
1523             break;
1524
1525         c = avcodec_alloc_context3(NULL);
1526         if (!c)
1527             return AVERROR(ENOMEM);
1528
1529         c->width        = width;
1530         c->height       = height;
1531         c->flags        = AV_CODEC_FLAG_QSCALE | AV_CODEC_FLAG_PSNR;
1532         c->flags       |= s->avctx->flags & AV_CODEC_FLAG_QPEL;
1533         c->mb_decision  = s->avctx->mb_decision;
1534         c->me_cmp       = s->avctx->me_cmp;
1535         c->mb_cmp       = s->avctx->mb_cmp;
1536         c->me_sub_cmp   = s->avctx->me_sub_cmp;
1537         c->pix_fmt      = AV_PIX_FMT_YUV420P;
1538         c->time_base    = s->avctx->time_base;
1539         c->max_b_frames = s->max_b_frames;
1540
1541         ret = avcodec_open2(c, codec, NULL);
1542         if (ret < 0)
1543             goto fail;
1544
1545         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1546         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1547
1548         out_size = encode_frame(c, s->tmp_frames[0]);
1549         if (out_size < 0) {
1550             ret = out_size;
1551             goto fail;
1552         }
1553
1554         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1555
1556         for (i = 0; i < s->max_b_frames + 1; i++) {
1557             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1558
1559             s->tmp_frames[i + 1]->pict_type = is_p ?
1560                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1561             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1562
1563             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1564             if (out_size < 0) {
1565                 ret = out_size;
1566                 goto fail;
1567             }
1568
1569             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1570         }
1571
1572         /* get the delayed frames */
1573         out_size = encode_frame(c, NULL);
1574         if (out_size < 0) {
1575             ret = out_size;
1576             goto fail;
1577         }
1578         rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1579
1580         rd += c->error[0] + c->error[1] + c->error[2];
1581
1582         if (rd < best_rd) {
1583             best_rd = rd;
1584             best_b_count = j;
1585         }
1586
1587 fail:
1588         avcodec_free_context(&c);
1589         if (ret < 0)
1590             return ret;
1591     }
1592
1593     return best_b_count;
1594 }
1595
1596 static int select_input_picture(MpegEncContext *s)
1597 {
1598     int i, ret;
1599
1600     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1601         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1602     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1603
1604     /* set next picture type & ordering */
1605     if (!s->reordered_input_picture[0] && s->input_picture[0]) {
1606         if (s->frame_skip_threshold || s->frame_skip_factor) {
1607             if (s->picture_in_gop_number < s->gop_size &&
1608                 s->next_picture_ptr &&
1609                 skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1610                 // FIXME check that the gop check above is +-1 correct
1611                 av_frame_unref(s->input_picture[0]->f);
1612
1613                 ff_vbv_update(s, 0);
1614
1615                 goto no_output_pic;
1616             }
1617         }
1618
1619         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1620             !s->next_picture_ptr || s->intra_only) {
1621             s->reordered_input_picture[0] = s->input_picture[0];
1622             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1623             s->reordered_input_picture[0]->f->coded_picture_number =
1624                 s->coded_picture_number++;
1625         } else {
1626             int b_frames = 0;
1627
1628             if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
1629                 for (i = 0; i < s->max_b_frames + 1; i++) {
1630                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1631
1632                     if (pict_num >= s->rc_context.num_entries)
1633                         break;
1634                     if (!s->input_picture[i]) {
1635                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1636                         break;
1637                     }
1638
1639                     s->input_picture[i]->f->pict_type =
1640                         s->rc_context.entry[pict_num].new_pict_type;
1641                 }
1642             }
1643
1644             if (s->b_frame_strategy == 0) {
1645                 b_frames = s->max_b_frames;
1646                 while (b_frames && !s->input_picture[b_frames])
1647                     b_frames--;
1648             } else if (s->b_frame_strategy == 1) {
1649                 for (i = 1; i < s->max_b_frames + 1; i++) {
1650                     if (s->input_picture[i] &&
1651                         s->input_picture[i]->b_frame_score == 0) {
1652                         s->input_picture[i]->b_frame_score =
1653                             get_intra_count(s,
1654                                             s->input_picture[i    ]->f->data[0],
1655                                             s->input_picture[i - 1]->f->data[0],
1656                                             s->linesize) + 1;
1657                     }
1658                 }
1659                 for (i = 0; i < s->max_b_frames + 1; i++) {
1660                     if (!s->input_picture[i] ||
1661                         s->input_picture[i]->b_frame_score - 1 >
1662                             s->mb_num / s->b_sensitivity)
1663                         break;
1664                 }
1665
1666                 b_frames = FFMAX(0, i - 1);
1667
1668                 /* reset scores */
1669                 for (i = 0; i < b_frames + 1; i++) {
1670                     s->input_picture[i]->b_frame_score = 0;
1671                 }
1672             } else if (s->b_frame_strategy == 2) {
1673                 b_frames = estimate_best_b_count(s);
1674                 if (b_frames < 0)
1675                     return b_frames;
1676             }
1677
1678             emms_c();
1679
1680             for (i = b_frames - 1; i >= 0; i--) {
1681                 int type = s->input_picture[i]->f->pict_type;
1682                 if (type && type != AV_PICTURE_TYPE_B)
1683                     b_frames = i;
1684             }
1685             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1686                 b_frames == s->max_b_frames) {
1687                 av_log(s->avctx, AV_LOG_ERROR,
1688                        "warning, too many B-frames in a row\n");
1689             }
1690
1691             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1692                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1693                     s->gop_size > s->picture_in_gop_number) {
1694                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1695                 } else {
1696                     if (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)
1697                         b_frames = 0;
1698                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1699                 }
1700             }
1701
1702             if ((s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP) && b_frames &&
1703                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1704                 b_frames--;
1705
1706             s->reordered_input_picture[0] = s->input_picture[b_frames];
1707             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1708                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1709             s->reordered_input_picture[0]->f->coded_picture_number =
1710                 s->coded_picture_number++;
1711             for (i = 0; i < b_frames; i++) {
1712                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1713                 s->reordered_input_picture[i + 1]->f->pict_type =
1714                     AV_PICTURE_TYPE_B;
1715                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1716                     s->coded_picture_number++;
1717             }
1718         }
1719     }
1720 no_output_pic:
1721     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1722
1723     if (s->reordered_input_picture[0]) {
1724         s->reordered_input_picture[0]->reference =
1725            s->reordered_input_picture[0]->f->pict_type !=
1726                AV_PICTURE_TYPE_B ? 3 : 0;
1727
1728         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->new_picture, s->reordered_input_picture[0])))
1729             return ret;
1730
1731         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1732             // input is a shared pix, so we can't modify it -> allocate a new
1733             // one & ensure that the shared one is reuseable
1734
1735             Picture *pic;
1736             int i = ff_find_unused_picture(s->avctx, s->picture, 0);
1737             if (i < 0)
1738                 return i;
1739             pic = &s->picture[i];
1740
1741             pic->reference = s->reordered_input_picture[0]->reference;
1742             if (alloc_picture(s, pic, 0) < 0) {
1743                 return -1;
1744             }
1745
1746             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1747             if (ret < 0)
1748                 return ret;
1749
1750             /* mark us unused / free shared pic */
1751             av_frame_unref(s->reordered_input_picture[0]->f);
1752             s->reordered_input_picture[0]->shared = 0;
1753
1754             s->current_picture_ptr = pic;
1755         } else {
1756             // input is not a shared pix -> reuse buffer for current_pix
1757             s->current_picture_ptr = s->reordered_input_picture[0];
1758             for (i = 0; i < 4; i++) {
1759                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1760             }
1761         }
1762         ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1763         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1764                                        s->current_picture_ptr)) < 0)
1765             return ret;
1766
1767         s->picture_number = s->new_picture.f->display_picture_number;
1768     }
1769     return 0;
1770 }
1771
1772 static void frame_end(MpegEncContext *s)
1773 {
1774     if (s->unrestricted_mv &&
1775         s->current_picture.reference &&
1776         !s->intra_only) {
1777         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1778         int hshift = desc->log2_chroma_w;
1779         int vshift = desc->log2_chroma_h;
1780         s->mpvencdsp.draw_edges(s->current_picture.f->data[0],
1781                                 s->current_picture.f->linesize[0],
1782                                 s->h_edge_pos, s->v_edge_pos,
1783                                 EDGE_WIDTH, EDGE_WIDTH,
1784                                 EDGE_TOP | EDGE_BOTTOM);
1785         s->mpvencdsp.draw_edges(s->current_picture.f->data[1],
1786                                 s->current_picture.f->linesize[1],
1787                                 s->h_edge_pos >> hshift,
1788                                 s->v_edge_pos >> vshift,
1789                                 EDGE_WIDTH >> hshift,
1790                                 EDGE_WIDTH >> vshift,
1791                                 EDGE_TOP | EDGE_BOTTOM);
1792         s->mpvencdsp.draw_edges(s->current_picture.f->data[2],
1793                                 s->current_picture.f->linesize[2],
1794                                 s->h_edge_pos >> hshift,
1795                                 s->v_edge_pos >> vshift,
1796                                 EDGE_WIDTH >> hshift,
1797                                 EDGE_WIDTH >> vshift,
1798                                 EDGE_TOP | EDGE_BOTTOM);
1799     }
1800
1801     emms_c();
1802
1803     s->last_pict_type                 = s->pict_type;
1804     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1805     if (s->pict_type!= AV_PICTURE_TYPE_B)
1806         s->last_non_b_pict_type = s->pict_type;
1807
1808 #if FF_API_CODED_FRAME
1809 FF_DISABLE_DEPRECATION_WARNINGS
1810     av_frame_unref(s->avctx->coded_frame);
1811     av_frame_copy_props(s->avctx->coded_frame, s->current_picture.f);
1812 FF_ENABLE_DEPRECATION_WARNINGS
1813 #endif
1814 #if FF_API_ERROR_FRAME
1815 FF_DISABLE_DEPRECATION_WARNINGS
1816     memcpy(s->current_picture.f->error, s->current_picture.encoding_error,
1817            sizeof(s->current_picture.encoding_error));
1818 FF_ENABLE_DEPRECATION_WARNINGS
1819 #endif
1820 }
1821
1822 static void update_noise_reduction(MpegEncContext *s)
1823 {
1824     int intra, i;
1825
1826     for (intra = 0; intra < 2; intra++) {
1827         if (s->dct_count[intra] > (1 << 16)) {
1828             for (i = 0; i < 64; i++) {
1829                 s->dct_error_sum[intra][i] >>= 1;
1830             }
1831             s->dct_count[intra] >>= 1;
1832         }
1833
1834         for (i = 0; i < 64; i++) {
1835             s->dct_offset[intra][i] = (s->noise_reduction *
1836                                        s->dct_count[intra] +
1837                                        s->dct_error_sum[intra][i] / 2) /
1838                                       (s->dct_error_sum[intra][i] + 1);
1839         }
1840     }
1841 }
1842
1843 static int frame_start(MpegEncContext *s)
1844 {
1845     int ret;
1846
1847     /* mark & release old frames */
1848     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1849         s->last_picture_ptr != s->next_picture_ptr &&
1850         s->last_picture_ptr->f->buf[0]) {
1851         ff_mpeg_unref_picture(s->avctx, s->last_picture_ptr);
1852     }
1853
1854     s->current_picture_ptr->f->pict_type = s->pict_type;
1855     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1856
1857     ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1858     if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1859                                    s->current_picture_ptr)) < 0)
1860         return ret;
1861
1862     if (s->pict_type != AV_PICTURE_TYPE_B) {
1863         s->last_picture_ptr = s->next_picture_ptr;
1864         if (!s->droppable)
1865             s->next_picture_ptr = s->current_picture_ptr;
1866     }
1867
1868     if (s->last_picture_ptr) {
1869         ff_mpeg_unref_picture(s->avctx, &s->last_picture);
1870         if (s->last_picture_ptr->f->buf[0] &&
1871             (ret = ff_mpeg_ref_picture(s->avctx, &s->last_picture,
1872                                        s->last_picture_ptr)) < 0)
1873             return ret;
1874     }
1875     if (s->next_picture_ptr) {
1876         ff_mpeg_unref_picture(s->avctx, &s->next_picture);
1877         if (s->next_picture_ptr->f->buf[0] &&
1878             (ret = ff_mpeg_ref_picture(s->avctx, &s->next_picture,
1879                                        s->next_picture_ptr)) < 0)
1880             return ret;
1881     }
1882
1883     if (s->picture_structure!= PICT_FRAME) {
1884         int i;
1885         for (i = 0; i < 4; i++) {
1886             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1887                 s->current_picture.f->data[i] +=
1888                     s->current_picture.f->linesize[i];
1889             }
1890             s->current_picture.f->linesize[i] *= 2;
1891             s->last_picture.f->linesize[i]    *= 2;
1892             s->next_picture.f->linesize[i]    *= 2;
1893         }
1894     }
1895
1896     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1897         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1898         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1899     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1900         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1901         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1902     } else {
1903         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1904         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1905     }
1906
1907     if (s->dct_error_sum) {
1908         av_assert2(s->noise_reduction && s->encoding);
1909         update_noise_reduction(s);
1910     }
1911
1912     return 0;
1913 }
1914
1915 int ff_mpv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1916                           const AVFrame *pic_arg, int *got_packet)
1917 {
1918     MpegEncContext *s = avctx->priv_data;
1919     int i, stuffing_count, ret;
1920     int context_count = s->slice_context_count;
1921
1922     s->vbv_ignore_qmax = 0;
1923
1924     s->picture_in_gop_number++;
1925
1926     if (load_input_picture(s, pic_arg) < 0)
1927         return -1;
1928
1929     if (select_input_picture(s) < 0) {
1930         return -1;
1931     }
1932
1933     /* output? */
1934     if (s->new_picture.f->data[0]) {
1935         int growing_buffer = context_count == 1 && !pkt->data && !s->data_partitioning;
1936         int pkt_size = growing_buffer ? FFMAX(s->mb_width*s->mb_height*64+10000, avctx->internal->byte_buffer_size) - AV_INPUT_BUFFER_PADDING_SIZE
1937                                               :
1938                                               s->mb_width*s->mb_height*(MAX_MB_BYTES+100)+10000;
1939         if ((ret = ff_alloc_packet2(avctx, pkt, pkt_size, 0)) < 0)
1940             return ret;
1941         if (s->mb_info) {
1942             s->mb_info_ptr = av_packet_new_side_data(pkt,
1943                                  AV_PKT_DATA_H263_MB_INFO,
1944                                  s->mb_width*s->mb_height*12);
1945             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1946         }
1947
1948         for (i = 0; i < context_count; i++) {
1949             int start_y = s->thread_context[i]->start_mb_y;
1950             int   end_y = s->thread_context[i]->  end_mb_y;
1951             int h       = s->mb_height;
1952             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1953             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1954
1955             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1956         }
1957
1958         s->pict_type = s->new_picture.f->pict_type;
1959         //emms_c();
1960         ret = frame_start(s);
1961         if (ret < 0)
1962             return ret;
1963 vbv_retry:
1964         ret = encode_picture(s, s->picture_number);
1965         if (growing_buffer) {
1966             av_assert0(s->pb.buf == avctx->internal->byte_buffer);
1967             pkt->data = s->pb.buf;
1968             pkt->size = avctx->internal->byte_buffer_size;
1969         }
1970         if (ret < 0)
1971             return -1;
1972
1973 #if FF_API_STAT_BITS
1974 FF_DISABLE_DEPRECATION_WARNINGS
1975         avctx->header_bits = s->header_bits;
1976         avctx->mv_bits     = s->mv_bits;
1977         avctx->misc_bits   = s->misc_bits;
1978         avctx->i_tex_bits  = s->i_tex_bits;
1979         avctx->p_tex_bits  = s->p_tex_bits;
1980         avctx->i_count     = s->i_count;
1981         // FIXME f/b_count in avctx
1982         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1983         avctx->skip_count  = s->skip_count;
1984 FF_ENABLE_DEPRECATION_WARNINGS
1985 #endif
1986
1987         frame_end(s);
1988
1989         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1990             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1991
1992         if (avctx->rc_buffer_size) {
1993             RateControlContext *rcc = &s->rc_context;
1994             int max_size = FFMAX(rcc->buffer_index * avctx->rc_max_available_vbv_use, rcc->buffer_index - 500);
1995             int hq = (s->avctx->mb_decision == FF_MB_DECISION_RD || s->avctx->trellis);
1996             int min_step = hq ? 1 : (1<<(FF_LAMBDA_SHIFT + 7))/139;
1997
1998             if (put_bits_count(&s->pb) > max_size &&
1999                 s->lambda < s->lmax) {
2000                 s->next_lambda = FFMAX(s->lambda + min_step, s->lambda *
2001                                        (s->qscale + 1) / s->qscale);
2002                 if (s->adaptive_quant) {
2003                     int i;
2004                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
2005                         s->lambda_table[i] =
2006                             FFMAX(s->lambda_table[i] + min_step,
2007                                   s->lambda_table[i] * (s->qscale + 1) /
2008                                   s->qscale);
2009                 }
2010                 s->mb_skipped = 0;        // done in frame_start()
2011                 // done in encode_picture() so we must undo it
2012                 if (s->pict_type == AV_PICTURE_TYPE_P) {
2013                     if (s->flipflop_rounding          ||
2014                         s->codec_id == AV_CODEC_ID_H263P ||
2015                         s->codec_id == AV_CODEC_ID_MPEG4)
2016                         s->no_rounding ^= 1;
2017                 }
2018                 if (s->pict_type != AV_PICTURE_TYPE_B) {
2019                     s->time_base       = s->last_time_base;
2020                     s->last_non_b_time = s->time - s->pp_time;
2021                 }
2022                 for (i = 0; i < context_count; i++) {
2023                     PutBitContext *pb = &s->thread_context[i]->pb;
2024                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
2025                 }
2026                 s->vbv_ignore_qmax = 1;
2027                 av_log(s->avctx, AV_LOG_VERBOSE, "reencoding frame due to VBV\n");
2028                 goto vbv_retry;
2029             }
2030
2031             av_assert0(s->avctx->rc_max_rate);
2032         }
2033
2034         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
2035             ff_write_pass1_stats(s);
2036
2037         for (i = 0; i < 4; i++) {
2038             s->current_picture_ptr->encoding_error[i] = s->current_picture.encoding_error[i];
2039             avctx->error[i] += s->current_picture_ptr->encoding_error[i];
2040         }
2041         ff_side_data_set_encoder_stats(pkt, s->current_picture.f->quality,
2042                                        s->current_picture_ptr->encoding_error,
2043                                        (s->avctx->flags&AV_CODEC_FLAG_PSNR) ? 4 : 0,
2044                                        s->pict_type);
2045
2046         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
2047             assert(put_bits_count(&s->pb) == s->header_bits + s->mv_bits +
2048                                              s->misc_bits + s->i_tex_bits +
2049                                              s->p_tex_bits);
2050         flush_put_bits(&s->pb);
2051         s->frame_bits  = put_bits_count(&s->pb);
2052
2053         stuffing_count = ff_vbv_update(s, s->frame_bits);
2054         s->stuffing_bits = 8*stuffing_count;
2055         if (stuffing_count) {
2056             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
2057                     stuffing_count + 50) {
2058                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
2059                 return -1;
2060             }
2061
2062             switch (s->codec_id) {
2063             case AV_CODEC_ID_MPEG1VIDEO:
2064             case AV_CODEC_ID_MPEG2VIDEO:
2065                 while (stuffing_count--) {
2066                     put_bits(&s->pb, 8, 0);
2067                 }
2068             break;
2069             case AV_CODEC_ID_MPEG4:
2070                 put_bits(&s->pb, 16, 0);
2071                 put_bits(&s->pb, 16, 0x1C3);
2072                 stuffing_count -= 4;
2073                 while (stuffing_count--) {
2074                     put_bits(&s->pb, 8, 0xFF);
2075                 }
2076             break;
2077             default:
2078                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
2079             }
2080             flush_put_bits(&s->pb);
2081             s->frame_bits  = put_bits_count(&s->pb);
2082         }
2083
2084         /* update MPEG-1/2 vbv_delay for CBR */
2085         if (s->avctx->rc_max_rate                          &&
2086             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
2087             s->out_format == FMT_MPEG1                     &&
2088             90000LL * (avctx->rc_buffer_size - 1) <=
2089                 s->avctx->rc_max_rate * 0xFFFFLL) {
2090             AVCPBProperties *props;
2091             size_t props_size;
2092
2093             int vbv_delay, min_delay;
2094             double inbits  = s->avctx->rc_max_rate *
2095                              av_q2d(s->avctx->time_base);
2096             int    minbits = s->frame_bits - 8 *
2097                              (s->vbv_delay_ptr - s->pb.buf - 1);
2098             double bits    = s->rc_context.buffer_index + minbits - inbits;
2099
2100             if (bits < 0)
2101                 av_log(s->avctx, AV_LOG_ERROR,
2102                        "Internal error, negative bits\n");
2103
2104             assert(s->repeat_first_field == 0);
2105
2106             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
2107             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
2108                         s->avctx->rc_max_rate;
2109
2110             vbv_delay = FFMAX(vbv_delay, min_delay);
2111
2112             av_assert0(vbv_delay < 0xFFFF);
2113
2114             s->vbv_delay_ptr[0] &= 0xF8;
2115             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
2116             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
2117             s->vbv_delay_ptr[2] &= 0x07;
2118             s->vbv_delay_ptr[2] |= vbv_delay << 3;
2119
2120             props = av_cpb_properties_alloc(&props_size);
2121             if (!props)
2122                 return AVERROR(ENOMEM);
2123             props->vbv_delay = vbv_delay * 300;
2124
2125             ret = av_packet_add_side_data(pkt, AV_PKT_DATA_CPB_PROPERTIES,
2126                                           (uint8_t*)props, props_size);
2127             if (ret < 0) {
2128                 av_freep(&props);
2129                 return ret;
2130             }
2131
2132 #if FF_API_VBV_DELAY
2133 FF_DISABLE_DEPRECATION_WARNINGS
2134             avctx->vbv_delay     = vbv_delay * 300;
2135 FF_ENABLE_DEPRECATION_WARNINGS
2136 #endif
2137         }
2138         s->total_bits     += s->frame_bits;
2139 #if FF_API_STAT_BITS
2140 FF_DISABLE_DEPRECATION_WARNINGS
2141         avctx->frame_bits  = s->frame_bits;
2142 FF_ENABLE_DEPRECATION_WARNINGS
2143 #endif
2144
2145
2146         pkt->pts = s->current_picture.f->pts;
2147         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
2148             if (!s->current_picture.f->coded_picture_number)
2149                 pkt->dts = pkt->pts - s->dts_delta;
2150             else
2151                 pkt->dts = s->reordered_pts;
2152             s->reordered_pts = pkt->pts;
2153         } else
2154             pkt->dts = pkt->pts;
2155         if (s->current_picture.f->key_frame)
2156             pkt->flags |= AV_PKT_FLAG_KEY;
2157         if (s->mb_info)
2158             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
2159     } else {
2160         s->frame_bits = 0;
2161     }
2162
2163     /* release non-reference frames */
2164     for (i = 0; i < MAX_PICTURE_COUNT; i++) {
2165         if (!s->picture[i].reference)
2166             ff_mpeg_unref_picture(s->avctx, &s->picture[i]);
2167     }
2168
2169     av_assert1((s->frame_bits & 7) == 0);
2170
2171     pkt->size = s->frame_bits / 8;
2172     *got_packet = !!pkt->size;
2173     return 0;
2174 }
2175
2176 static inline void dct_single_coeff_elimination(MpegEncContext *s,
2177                                                 int n, int threshold)
2178 {
2179     static const char tab[64] = {
2180         3, 2, 2, 1, 1, 1, 1, 1,
2181         1, 1, 1, 1, 1, 1, 1, 1,
2182         1, 1, 1, 1, 1, 1, 1, 1,
2183         0, 0, 0, 0, 0, 0, 0, 0,
2184         0, 0, 0, 0, 0, 0, 0, 0,
2185         0, 0, 0, 0, 0, 0, 0, 0,
2186         0, 0, 0, 0, 0, 0, 0, 0,
2187         0, 0, 0, 0, 0, 0, 0, 0
2188     };
2189     int score = 0;
2190     int run = 0;
2191     int i;
2192     int16_t *block = s->block[n];
2193     const int last_index = s->block_last_index[n];
2194     int skip_dc;
2195
2196     if (threshold < 0) {
2197         skip_dc = 0;
2198         threshold = -threshold;
2199     } else
2200         skip_dc = 1;
2201
2202     /* Are all we could set to zero already zero? */
2203     if (last_index <= skip_dc - 1)
2204         return;
2205
2206     for (i = 0; i <= last_index; i++) {
2207         const int j = s->intra_scantable.permutated[i];
2208         const int level = FFABS(block[j]);
2209         if (level == 1) {
2210             if (skip_dc && i == 0)
2211                 continue;
2212             score += tab[run];
2213             run = 0;
2214         } else if (level > 1) {
2215             return;
2216         } else {
2217             run++;
2218         }
2219     }
2220     if (score >= threshold)
2221         return;
2222     for (i = skip_dc; i <= last_index; i++) {
2223         const int j = s->intra_scantable.permutated[i];
2224         block[j] = 0;
2225     }
2226     if (block[0])
2227         s->block_last_index[n] = 0;
2228     else
2229         s->block_last_index[n] = -1;
2230 }
2231
2232 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
2233                                int last_index)
2234 {
2235     int i;
2236     const int maxlevel = s->max_qcoeff;
2237     const int minlevel = s->min_qcoeff;
2238     int overflow = 0;
2239
2240     if (s->mb_intra) {
2241         i = 1; // skip clipping of intra dc
2242     } else
2243         i = 0;
2244
2245     for (; i <= last_index; i++) {
2246         const int j = s->intra_scantable.permutated[i];
2247         int level = block[j];
2248
2249         if (level > maxlevel) {
2250             level = maxlevel;
2251             overflow++;
2252         } else if (level < minlevel) {
2253             level = minlevel;
2254             overflow++;
2255         }
2256
2257         block[j] = level;
2258     }
2259
2260     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
2261         av_log(s->avctx, AV_LOG_INFO,
2262                "warning, clipping %d dct coefficients to %d..%d\n",
2263                overflow, minlevel, maxlevel);
2264 }
2265
2266 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
2267 {
2268     int x, y;
2269     // FIXME optimize
2270     for (y = 0; y < 8; y++) {
2271         for (x = 0; x < 8; x++) {
2272             int x2, y2;
2273             int sum = 0;
2274             int sqr = 0;
2275             int count = 0;
2276
2277             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
2278                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
2279                     int v = ptr[x2 + y2 * stride];
2280                     sum += v;
2281                     sqr += v * v;
2282                     count++;
2283                 }
2284             }
2285             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
2286         }
2287     }
2288 }
2289
2290 static av_always_inline void encode_mb_internal(MpegEncContext *s,
2291                                                 int motion_x, int motion_y,
2292                                                 int mb_block_height,
2293                                                 int mb_block_width,
2294                                                 int mb_block_count)
2295 {
2296     int16_t weight[12][64];
2297     int16_t orig[12][64];
2298     const int mb_x = s->mb_x;
2299     const int mb_y = s->mb_y;
2300     int i;
2301     int skip_dct[12];
2302     int dct_offset = s->linesize * 8; // default for progressive frames
2303     int uv_dct_offset = s->uvlinesize * 8;
2304     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
2305     ptrdiff_t wrap_y, wrap_c;
2306
2307     for (i = 0; i < mb_block_count; i++)
2308         skip_dct[i] = s->skipdct;
2309
2310     if (s->adaptive_quant) {
2311         const int last_qp = s->qscale;
2312         const int mb_xy = mb_x + mb_y * s->mb_stride;
2313
2314         s->lambda = s->lambda_table[mb_xy];
2315         update_qscale(s);
2316
2317         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
2318             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
2319             s->dquant = s->qscale - last_qp;
2320
2321             if (s->out_format == FMT_H263) {
2322                 s->dquant = av_clip(s->dquant, -2, 2);
2323
2324                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
2325                     if (!s->mb_intra) {
2326                         if (s->pict_type == AV_PICTURE_TYPE_B) {
2327                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
2328                                 s->dquant = 0;
2329                         }
2330                         if (s->mv_type == MV_TYPE_8X8)
2331                             s->dquant = 0;
2332                     }
2333                 }
2334             }
2335         }
2336         ff_set_qscale(s, last_qp + s->dquant);
2337     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
2338         ff_set_qscale(s, s->qscale + s->dquant);
2339
2340     wrap_y = s->linesize;
2341     wrap_c = s->uvlinesize;
2342     ptr_y  = s->new_picture.f->data[0] +
2343              (mb_y * 16 * wrap_y)              + mb_x * 16;
2344     ptr_cb = s->new_picture.f->data[1] +
2345              (mb_y * mb_block_height * wrap_c) + mb_x * mb_block_width;
2346     ptr_cr = s->new_picture.f->data[2] +
2347              (mb_y * mb_block_height * wrap_c) + mb_x * mb_block_width;
2348
2349     if((mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) && s->codec_id != AV_CODEC_ID_AMV){
2350         uint8_t *ebuf = s->sc.edge_emu_buffer + 38 * wrap_y;
2351         int cw = (s->width  + s->chroma_x_shift) >> s->chroma_x_shift;
2352         int ch = (s->height + s->chroma_y_shift) >> s->chroma_y_shift;
2353         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
2354                                  wrap_y, wrap_y,
2355                                  16, 16, mb_x * 16, mb_y * 16,
2356                                  s->width, s->height);
2357         ptr_y = ebuf;
2358         s->vdsp.emulated_edge_mc(ebuf + 16 * wrap_y, ptr_cb,
2359                                  wrap_c, wrap_c,
2360                                  mb_block_width, mb_block_height,
2361                                  mb_x * mb_block_width, mb_y * mb_block_height,
2362                                  cw, ch);
2363         ptr_cb = ebuf + 16 * wrap_y;
2364         s->vdsp.emulated_edge_mc(ebuf + 16 * wrap_y + 16, ptr_cr,
2365                                  wrap_c, wrap_c,
2366                                  mb_block_width, mb_block_height,
2367                                  mb_x * mb_block_width, mb_y * mb_block_height,
2368                                  cw, ch);
2369         ptr_cr = ebuf + 16 * wrap_y + 16;
2370     }
2371
2372     if (s->mb_intra) {
2373         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2374             int progressive_score, interlaced_score;
2375
2376             s->interlaced_dct = 0;
2377             progressive_score = s->mecc.ildct_cmp[4](s, ptr_y, NULL, wrap_y, 8) +
2378                                 s->mecc.ildct_cmp[4](s, ptr_y + wrap_y * 8,
2379                                                      NULL, wrap_y, 8) - 400;
2380
2381             if (progressive_score > 0) {
2382                 interlaced_score = s->mecc.ildct_cmp[4](s, ptr_y,
2383                                                         NULL, wrap_y * 2, 8) +
2384                                    s->mecc.ildct_cmp[4](s, ptr_y + wrap_y,
2385                                                         NULL, wrap_y * 2, 8);
2386                 if (progressive_score > interlaced_score) {
2387                     s->interlaced_dct = 1;
2388
2389                     dct_offset = wrap_y;
2390                     uv_dct_offset = wrap_c;
2391                     wrap_y <<= 1;
2392                     if (s->chroma_format == CHROMA_422 ||
2393                         s->chroma_format == CHROMA_444)
2394                         wrap_c <<= 1;
2395                 }
2396             }
2397         }
2398
2399         s->pdsp.get_pixels(s->block[0], ptr_y,                  wrap_y);
2400         s->pdsp.get_pixels(s->block[1], ptr_y + 8,              wrap_y);
2401         s->pdsp.get_pixels(s->block[2], ptr_y + dct_offset,     wrap_y);
2402         s->pdsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
2403
2404         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2405             skip_dct[4] = 1;
2406             skip_dct[5] = 1;
2407         } else {
2408             s->pdsp.get_pixels(s->block[4], ptr_cb, wrap_c);
2409             s->pdsp.get_pixels(s->block[5], ptr_cr, wrap_c);
2410             if (!s->chroma_y_shift && s->chroma_x_shift) { /* 422 */
2411                 s->pdsp.get_pixels(s->block[6], ptr_cb + uv_dct_offset, wrap_c);
2412                 s->pdsp.get_pixels(s->block[7], ptr_cr + uv_dct_offset, wrap_c);
2413             } else if (!s->chroma_y_shift && !s->chroma_x_shift) { /* 444 */
2414                 s->pdsp.get_pixels(s->block[ 6], ptr_cb + 8, wrap_c);
2415                 s->pdsp.get_pixels(s->block[ 7], ptr_cr + 8, wrap_c);
2416                 s->pdsp.get_pixels(s->block[ 8], ptr_cb + uv_dct_offset, wrap_c);
2417                 s->pdsp.get_pixels(s->block[ 9], ptr_cr + uv_dct_offset, wrap_c);
2418                 s->pdsp.get_pixels(s->block[10], ptr_cb + uv_dct_offset + 8, wrap_c);
2419                 s->pdsp.get_pixels(s->block[11], ptr_cr + uv_dct_offset + 8, wrap_c);
2420             }
2421         }
2422     } else {
2423         op_pixels_func (*op_pix)[4];
2424         qpel_mc_func (*op_qpix)[16];
2425         uint8_t *dest_y, *dest_cb, *dest_cr;
2426
2427         dest_y  = s->dest[0];
2428         dest_cb = s->dest[1];
2429         dest_cr = s->dest[2];
2430
2431         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
2432             op_pix  = s->hdsp.put_pixels_tab;
2433             op_qpix = s->qdsp.put_qpel_pixels_tab;
2434         } else {
2435             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
2436             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
2437         }
2438
2439         if (s->mv_dir & MV_DIR_FORWARD) {
2440             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 0,
2441                           s->last_picture.f->data,
2442                           op_pix, op_qpix);
2443             op_pix  = s->hdsp.avg_pixels_tab;
2444             op_qpix = s->qdsp.avg_qpel_pixels_tab;
2445         }
2446         if (s->mv_dir & MV_DIR_BACKWARD) {
2447             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 1,
2448                           s->next_picture.f->data,
2449                           op_pix, op_qpix);
2450         }
2451
2452         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2453             int progressive_score, interlaced_score;
2454
2455             s->interlaced_dct = 0;
2456             progressive_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y, wrap_y, 8) +
2457                                 s->mecc.ildct_cmp[0](s, dest_y + wrap_y * 8,
2458                                                      ptr_y + wrap_y * 8,
2459                                                      wrap_y, 8) - 400;
2460
2461             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2462                 progressive_score -= 400;
2463
2464             if (progressive_score > 0) {
2465                 interlaced_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y,
2466                                                         wrap_y * 2, 8) +
2467                                    s->mecc.ildct_cmp[0](s, dest_y + wrap_y,
2468                                                         ptr_y + wrap_y,
2469                                                         wrap_y * 2, 8);
2470
2471                 if (progressive_score > interlaced_score) {
2472                     s->interlaced_dct = 1;
2473
2474                     dct_offset = wrap_y;
2475                     uv_dct_offset = wrap_c;
2476                     wrap_y <<= 1;
2477                     if (s->chroma_format == CHROMA_422)
2478                         wrap_c <<= 1;
2479                 }
2480             }
2481         }
2482
2483         s->pdsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2484         s->pdsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2485         s->pdsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2486                             dest_y + dct_offset, wrap_y);
2487         s->pdsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2488                             dest_y + dct_offset + 8, wrap_y);
2489
2490         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2491             skip_dct[4] = 1;
2492             skip_dct[5] = 1;
2493         } else {
2494             s->pdsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2495             s->pdsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2496             if (!s->chroma_y_shift) { /* 422 */
2497                 s->pdsp.diff_pixels(s->block[6], ptr_cb + uv_dct_offset,
2498                                     dest_cb + uv_dct_offset, wrap_c);
2499                 s->pdsp.diff_pixels(s->block[7], ptr_cr + uv_dct_offset,
2500                                     dest_cr + uv_dct_offset, wrap_c);
2501             }
2502         }
2503         /* pre quantization */
2504         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2505                 2 * s->qscale * s->qscale) {
2506             // FIXME optimize
2507             if (s->mecc.sad[1](NULL, ptr_y, dest_y, wrap_y, 8) < 20 * s->qscale)
2508                 skip_dct[0] = 1;
2509             if (s->mecc.sad[1](NULL, ptr_y + 8, dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2510                 skip_dct[1] = 1;
2511             if (s->mecc.sad[1](NULL, ptr_y + dct_offset, dest_y + dct_offset,
2512                                wrap_y, 8) < 20 * s->qscale)
2513                 skip_dct[2] = 1;
2514             if (s->mecc.sad[1](NULL, ptr_y + dct_offset + 8, dest_y + dct_offset + 8,
2515                                wrap_y, 8) < 20 * s->qscale)
2516                 skip_dct[3] = 1;
2517             if (s->mecc.sad[1](NULL, ptr_cb, dest_cb, wrap_c, 8) < 20 * s->qscale)
2518                 skip_dct[4] = 1;
2519             if (s->mecc.sad[1](NULL, ptr_cr, dest_cr, wrap_c, 8) < 20 * s->qscale)
2520                 skip_dct[5] = 1;
2521             if (!s->chroma_y_shift) { /* 422 */
2522                 if (s->mecc.sad[1](NULL, ptr_cb + uv_dct_offset,
2523                                    dest_cb + uv_dct_offset,
2524                                    wrap_c, 8) < 20 * s->qscale)
2525                     skip_dct[6] = 1;
2526                 if (s->mecc.sad[1](NULL, ptr_cr + uv_dct_offset,
2527                                    dest_cr + uv_dct_offset,
2528                                    wrap_c, 8) < 20 * s->qscale)
2529                     skip_dct[7] = 1;
2530             }
2531         }
2532     }
2533
2534     if (s->quantizer_noise_shaping) {
2535         if (!skip_dct[0])
2536             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2537         if (!skip_dct[1])
2538             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2539         if (!skip_dct[2])
2540             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2541         if (!skip_dct[3])
2542             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2543         if (!skip_dct[4])
2544             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2545         if (!skip_dct[5])
2546             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2547         if (!s->chroma_y_shift) { /* 422 */
2548             if (!skip_dct[6])
2549                 get_visual_weight(weight[6], ptr_cb + uv_dct_offset,
2550                                   wrap_c);
2551             if (!skip_dct[7])
2552                 get_visual_weight(weight[7], ptr_cr + uv_dct_offset,
2553                                   wrap_c);
2554         }
2555         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2556     }
2557
2558     /* DCT & quantize */
2559     av_assert2(s->out_format != FMT_MJPEG || s->qscale == 8);
2560     {
2561         for (i = 0; i < mb_block_count; i++) {
2562             if (!skip_dct[i]) {
2563                 int overflow;
2564                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2565                 // FIXME we could decide to change to quantizer instead of
2566                 // clipping
2567                 // JS: I don't think that would be a good idea it could lower
2568                 //     quality instead of improve it. Just INTRADC clipping
2569                 //     deserves changes in quantizer
2570                 if (overflow)
2571                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2572             } else
2573                 s->block_last_index[i] = -1;
2574         }
2575         if (s->quantizer_noise_shaping) {
2576             for (i = 0; i < mb_block_count; i++) {
2577                 if (!skip_dct[i]) {
2578                     s->block_last_index[i] =
2579                         dct_quantize_refine(s, s->block[i], weight[i],
2580                                             orig[i], i, s->qscale);
2581                 }
2582             }
2583         }
2584
2585         if (s->luma_elim_threshold && !s->mb_intra)
2586             for (i = 0; i < 4; i++)
2587                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2588         if (s->chroma_elim_threshold && !s->mb_intra)
2589             for (i = 4; i < mb_block_count; i++)
2590                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2591
2592         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2593             for (i = 0; i < mb_block_count; i++) {
2594                 if (s->block_last_index[i] == -1)
2595                     s->coded_score[i] = INT_MAX / 256;
2596             }
2597         }
2598     }
2599
2600     if ((s->avctx->flags & AV_CODEC_FLAG_GRAY) && s->mb_intra) {
2601         s->block_last_index[4] =
2602         s->block_last_index[5] = 0;
2603         s->block[4][0] =
2604         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2605         if (!s->chroma_y_shift) { /* 422 / 444 */
2606             for (i=6; i<12; i++) {
2607                 s->block_last_index[i] = 0;
2608                 s->block[i][0] = s->block[4][0];
2609             }
2610         }
2611     }
2612
2613     // non c quantize code returns incorrect block_last_index FIXME
2614     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2615         for (i = 0; i < mb_block_count; i++) {
2616             int j;
2617             if (s->block_last_index[i] > 0) {
2618                 for (j = 63; j > 0; j--) {
2619                     if (s->block[i][s->intra_scantable.permutated[j]])
2620                         break;
2621                 }
2622                 s->block_last_index[i] = j;
2623             }
2624         }
2625     }
2626
2627     /* huffman encode */
2628     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2629     case AV_CODEC_ID_MPEG1VIDEO:
2630     case AV_CODEC_ID_MPEG2VIDEO:
2631         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2632             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2633         break;
2634     case AV_CODEC_ID_MPEG4:
2635         if (CONFIG_MPEG4_ENCODER)
2636             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2637         break;
2638     case AV_CODEC_ID_MSMPEG4V2:
2639     case AV_CODEC_ID_MSMPEG4V3:
2640     case AV_CODEC_ID_WMV1:
2641         if (CONFIG_MSMPEG4_ENCODER)
2642             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2643         break;
2644     case AV_CODEC_ID_WMV2:
2645         if (CONFIG_WMV2_ENCODER)
2646             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2647         break;
2648     case AV_CODEC_ID_H261:
2649         if (CONFIG_H261_ENCODER)
2650             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2651         break;
2652     case AV_CODEC_ID_H263:
2653     case AV_CODEC_ID_H263P:
2654     case AV_CODEC_ID_FLV1:
2655     case AV_CODEC_ID_RV10:
2656     case AV_CODEC_ID_RV20:
2657         if (CONFIG_H263_ENCODER)
2658             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2659         break;
2660     case AV_CODEC_ID_MJPEG:
2661     case AV_CODEC_ID_AMV:
2662         if (CONFIG_MJPEG_ENCODER)
2663             ff_mjpeg_encode_mb(s, s->block);
2664         break;
2665     default:
2666         av_assert1(0);
2667     }
2668 }
2669
2670 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2671 {
2672     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 8, 6);
2673     else if (s->chroma_format == CHROMA_422) encode_mb_internal(s, motion_x, motion_y, 16, 8, 8);
2674     else encode_mb_internal(s, motion_x, motion_y, 16, 16, 12);
2675 }
2676
2677 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2678     int i;
2679
2680     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2681
2682     /* MPEG-1 */
2683     d->mb_skip_run= s->mb_skip_run;
2684     for(i=0; i<3; i++)
2685         d->last_dc[i] = s->last_dc[i];
2686
2687     /* statistics */
2688     d->mv_bits= s->mv_bits;
2689     d->i_tex_bits= s->i_tex_bits;
2690     d->p_tex_bits= s->p_tex_bits;
2691     d->i_count= s->i_count;
2692     d->f_count= s->f_count;
2693     d->b_count= s->b_count;
2694     d->skip_count= s->skip_count;
2695     d->misc_bits= s->misc_bits;
2696     d->last_bits= 0;
2697
2698     d->mb_skipped= 0;
2699     d->qscale= s->qscale;
2700     d->dquant= s->dquant;
2701
2702     d->esc3_level_length= s->esc3_level_length;
2703 }
2704
2705 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2706     int i;
2707
2708     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2709     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2710
2711     /* MPEG-1 */
2712     d->mb_skip_run= s->mb_skip_run;
2713     for(i=0; i<3; i++)
2714         d->last_dc[i] = s->last_dc[i];
2715
2716     /* statistics */
2717     d->mv_bits= s->mv_bits;
2718     d->i_tex_bits= s->i_tex_bits;
2719     d->p_tex_bits= s->p_tex_bits;
2720     d->i_count= s->i_count;
2721     d->f_count= s->f_count;
2722     d->b_count= s->b_count;
2723     d->skip_count= s->skip_count;
2724     d->misc_bits= s->misc_bits;
2725
2726     d->mb_intra= s->mb_intra;
2727     d->mb_skipped= s->mb_skipped;
2728     d->mv_type= s->mv_type;
2729     d->mv_dir= s->mv_dir;
2730     d->pb= s->pb;
2731     if(s->data_partitioning){
2732         d->pb2= s->pb2;
2733         d->tex_pb= s->tex_pb;
2734     }
2735     d->block= s->block;
2736     for(i=0; i<8; i++)
2737         d->block_last_index[i]= s->block_last_index[i];
2738     d->interlaced_dct= s->interlaced_dct;
2739     d->qscale= s->qscale;
2740
2741     d->esc3_level_length= s->esc3_level_length;
2742 }
2743
2744 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2745                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2746                            int *dmin, int *next_block, int motion_x, int motion_y)
2747 {
2748     int score;
2749     uint8_t *dest_backup[3];
2750
2751     copy_context_before_encode(s, backup, type);
2752
2753     s->block= s->blocks[*next_block];
2754     s->pb= pb[*next_block];
2755     if(s->data_partitioning){
2756         s->pb2   = pb2   [*next_block];
2757         s->tex_pb= tex_pb[*next_block];
2758     }
2759
2760     if(*next_block){
2761         memcpy(dest_backup, s->dest, sizeof(s->dest));
2762         s->dest[0] = s->sc.rd_scratchpad;
2763         s->dest[1] = s->sc.rd_scratchpad + 16*s->linesize;
2764         s->dest[2] = s->sc.rd_scratchpad + 16*s->linesize + 8;
2765         av_assert0(s->linesize >= 32); //FIXME
2766     }
2767
2768     encode_mb(s, motion_x, motion_y);
2769
2770     score= put_bits_count(&s->pb);
2771     if(s->data_partitioning){
2772         score+= put_bits_count(&s->pb2);
2773         score+= put_bits_count(&s->tex_pb);
2774     }
2775
2776     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2777         ff_mpv_reconstruct_mb(s, s->block);
2778
2779         score *= s->lambda2;
2780         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2781     }
2782
2783     if(*next_block){
2784         memcpy(s->dest, dest_backup, sizeof(s->dest));
2785     }
2786
2787     if(score<*dmin){
2788         *dmin= score;
2789         *next_block^=1;
2790
2791         copy_context_after_encode(best, s, type);
2792     }
2793 }
2794
2795 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2796     uint32_t *sq = ff_square_tab + 256;
2797     int acc=0;
2798     int x,y;
2799
2800     if(w==16 && h==16)
2801         return s->mecc.sse[0](NULL, src1, src2, stride, 16);
2802     else if(w==8 && h==8)
2803         return s->mecc.sse[1](NULL, src1, src2, stride, 8);
2804
2805     for(y=0; y<h; y++){
2806         for(x=0; x<w; x++){
2807             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2808         }
2809     }
2810
2811     av_assert2(acc>=0);
2812
2813     return acc;
2814 }
2815
2816 static int sse_mb(MpegEncContext *s){
2817     int w= 16;
2818     int h= 16;
2819
2820     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2821     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2822
2823     if(w==16 && h==16)
2824       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2825         return s->mecc.nsse[0](s, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2826                s->mecc.nsse[1](s, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2827                s->mecc.nsse[1](s, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2828       }else{
2829         return s->mecc.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2830                s->mecc.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2831                s->mecc.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2832       }
2833     else
2834         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2835                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2836                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2837 }
2838
2839 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2840     MpegEncContext *s= *(void**)arg;
2841
2842
2843     s->me.pre_pass=1;
2844     s->me.dia_size= s->avctx->pre_dia_size;
2845     s->first_slice_line=1;
2846     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2847         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2848             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2849         }
2850         s->first_slice_line=0;
2851     }
2852
2853     s->me.pre_pass=0;
2854
2855     return 0;
2856 }
2857
2858 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2859     MpegEncContext *s= *(void**)arg;
2860
2861     ff_check_alignment();
2862
2863     s->me.dia_size= s->avctx->dia_size;
2864     s->first_slice_line=1;
2865     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2866         s->mb_x=0; //for block init below
2867         ff_init_block_index(s);
2868         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2869             s->block_index[0]+=2;
2870             s->block_index[1]+=2;
2871             s->block_index[2]+=2;
2872             s->block_index[3]+=2;
2873
2874             /* compute motion vector & mb_type and store in context */
2875             if(s->pict_type==AV_PICTURE_TYPE_B)
2876                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2877             else
2878                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2879         }
2880         s->first_slice_line=0;
2881     }
2882     return 0;
2883 }
2884
2885 static int mb_var_thread(AVCodecContext *c, void *arg){
2886     MpegEncContext *s= *(void**)arg;
2887     int mb_x, mb_y;
2888
2889     ff_check_alignment();
2890
2891     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2892         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2893             int xx = mb_x * 16;
2894             int yy = mb_y * 16;
2895             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2896             int varc;
2897             int sum = s->mpvencdsp.pix_sum(pix, s->linesize);
2898
2899             varc = (s->mpvencdsp.pix_norm1(pix, s->linesize) -
2900                     (((unsigned) sum * sum) >> 8) + 500 + 128) >> 8;
2901
2902             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2903             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2904             s->me.mb_var_sum_temp    += varc;
2905         }
2906     }
2907     return 0;
2908 }
2909
2910 static void write_slice_end(MpegEncContext *s){
2911     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2912         if(s->partitioned_frame){
2913             ff_mpeg4_merge_partitions(s);
2914         }
2915
2916         ff_mpeg4_stuffing(&s->pb);
2917     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2918         ff_mjpeg_encode_stuffing(s);
2919     }
2920
2921     avpriv_align_put_bits(&s->pb);
2922     flush_put_bits(&s->pb);
2923
2924     if ((s->avctx->flags & AV_CODEC_FLAG_PASS1) && !s->partitioned_frame)
2925         s->misc_bits+= get_bits_diff(s);
2926 }
2927
2928 static void write_mb_info(MpegEncContext *s)
2929 {
2930     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2931     int offset = put_bits_count(&s->pb);
2932     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2933     int gobn = s->mb_y / s->gob_index;
2934     int pred_x, pred_y;
2935     if (CONFIG_H263_ENCODER)
2936         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2937     bytestream_put_le32(&ptr, offset);
2938     bytestream_put_byte(&ptr, s->qscale);
2939     bytestream_put_byte(&ptr, gobn);
2940     bytestream_put_le16(&ptr, mba);
2941     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2942     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2943     /* 4MV not implemented */
2944     bytestream_put_byte(&ptr, 0); /* hmv2 */
2945     bytestream_put_byte(&ptr, 0); /* vmv2 */
2946 }
2947
2948 static void update_mb_info(MpegEncContext *s, int startcode)
2949 {
2950     if (!s->mb_info)
2951         return;
2952     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2953         s->mb_info_size += 12;
2954         s->prev_mb_info = s->last_mb_info;
2955     }
2956     if (startcode) {
2957         s->prev_mb_info = put_bits_count(&s->pb)/8;
2958         /* This might have incremented mb_info_size above, and we return without
2959          * actually writing any info into that slot yet. But in that case,
2960          * this will be called again at the start of the after writing the
2961          * start code, actually writing the mb info. */
2962         return;
2963     }
2964
2965     s->last_mb_info = put_bits_count(&s->pb)/8;
2966     if (!s->mb_info_size)
2967         s->mb_info_size += 12;
2968     write_mb_info(s);
2969 }
2970
2971 int ff_mpv_reallocate_putbitbuffer(MpegEncContext *s, size_t threshold, size_t size_increase)
2972 {
2973     if (   s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < threshold
2974         && s->slice_context_count == 1
2975         && s->pb.buf == s->avctx->internal->byte_buffer) {
2976         int lastgob_pos = s->ptr_lastgob - s->pb.buf;
2977         int vbv_pos     = s->vbv_delay_ptr - s->pb.buf;
2978
2979         uint8_t *new_buffer = NULL;
2980         int new_buffer_size = 0;
2981
2982         if ((s->avctx->internal->byte_buffer_size + size_increase) >= INT_MAX/8) {
2983             av_log(s->avctx, AV_LOG_ERROR, "Cannot reallocate putbit buffer\n");
2984             return AVERROR(ENOMEM);
2985         }
2986
2987         emms_c();
2988
2989         av_fast_padded_malloc(&new_buffer, &new_buffer_size,
2990                               s->avctx->internal->byte_buffer_size + size_increase);
2991         if (!new_buffer)
2992             return AVERROR(ENOMEM);
2993
2994         memcpy(new_buffer, s->avctx->internal->byte_buffer, s->avctx->internal->byte_buffer_size);
2995         av_free(s->avctx->internal->byte_buffer);
2996         s->avctx->internal->byte_buffer      = new_buffer;
2997         s->avctx->internal->byte_buffer_size = new_buffer_size;
2998         rebase_put_bits(&s->pb, new_buffer, new_buffer_size);
2999         s->ptr_lastgob   = s->pb.buf + lastgob_pos;
3000         s->vbv_delay_ptr = s->pb.buf + vbv_pos;
3001     }
3002     if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < threshold)
3003         return AVERROR(EINVAL);
3004     return 0;
3005 }
3006
3007 static int encode_thread(AVCodecContext *c, void *arg){
3008     MpegEncContext *s= *(void**)arg;
3009     int mb_x, mb_y;
3010     int chr_h= 16>>s->chroma_y_shift;
3011     int i, j;
3012     MpegEncContext best_s = { 0 }, backup_s;
3013     uint8_t bit_buf[2][MAX_MB_BYTES];
3014     uint8_t bit_buf2[2][MAX_MB_BYTES];
3015     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
3016     PutBitContext pb[2], pb2[2], tex_pb[2];
3017
3018     ff_check_alignment();
3019
3020     for(i=0; i<2; i++){
3021         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
3022         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
3023         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
3024     }
3025
3026     s->last_bits= put_bits_count(&s->pb);
3027     s->mv_bits=0;
3028     s->misc_bits=0;
3029     s->i_tex_bits=0;
3030     s->p_tex_bits=0;
3031     s->i_count=0;
3032     s->f_count=0;
3033     s->b_count=0;
3034     s->skip_count=0;
3035
3036     for(i=0; i<3; i++){
3037         /* init last dc values */
3038         /* note: quant matrix value (8) is implied here */
3039         s->last_dc[i] = 128 << s->intra_dc_precision;
3040
3041         s->current_picture.encoding_error[i] = 0;
3042     }
3043     if(s->codec_id==AV_CODEC_ID_AMV){
3044         s->last_dc[0] = 128*8/13;
3045         s->last_dc[1] = 128*8/14;
3046         s->last_dc[2] = 128*8/14;
3047     }
3048     s->mb_skip_run = 0;
3049     memset(s->last_mv, 0, sizeof(s->last_mv));
3050
3051     s->last_mv_dir = 0;
3052
3053     switch(s->codec_id){
3054     case AV_CODEC_ID_H263:
3055     case AV_CODEC_ID_H263P:
3056     case AV_CODEC_ID_FLV1:
3057         if (CONFIG_H263_ENCODER)
3058             s->gob_index = H263_GOB_HEIGHT(s->height);
3059         break;
3060     case AV_CODEC_ID_MPEG4:
3061         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
3062             ff_mpeg4_init_partitions(s);
3063         break;
3064     }
3065
3066     s->resync_mb_x=0;
3067     s->resync_mb_y=0;
3068     s->first_slice_line = 1;
3069     s->ptr_lastgob = s->pb.buf;
3070     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
3071         s->mb_x=0;
3072         s->mb_y= mb_y;
3073
3074         ff_set_qscale(s, s->qscale);
3075         ff_init_block_index(s);
3076
3077         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
3078             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
3079             int mb_type= s->mb_type[xy];
3080 //            int d;
3081             int dmin= INT_MAX;
3082             int dir;
3083             int size_increase =  s->avctx->internal->byte_buffer_size/4
3084                                + s->mb_width*MAX_MB_BYTES;
3085
3086             ff_mpv_reallocate_putbitbuffer(s, MAX_MB_BYTES, size_increase);
3087             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
3088                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
3089                 return -1;
3090             }
3091             if(s->data_partitioning){
3092                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
3093                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
3094                     av_log(s->avctx, AV_LOG_ERROR, "encoded partitioned frame too large\n");
3095                     return -1;
3096                 }
3097             }
3098
3099             s->mb_x = mb_x;
3100             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
3101             ff_update_block_index(s);
3102
3103             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
3104                 ff_h261_reorder_mb_index(s);
3105                 xy= s->mb_y*s->mb_stride + s->mb_x;
3106                 mb_type= s->mb_type[xy];
3107             }
3108
3109             /* write gob / video packet header  */
3110             if(s->rtp_mode){
3111                 int current_packet_size, is_gob_start;
3112
3113                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
3114
3115                 is_gob_start = s->rtp_payload_size &&
3116                                current_packet_size >= s->rtp_payload_size &&
3117                                mb_y + mb_x > 0;
3118
3119                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
3120
3121                 switch(s->codec_id){
3122                 case AV_CODEC_ID_H263:
3123                 case AV_CODEC_ID_H263P:
3124                     if(!s->h263_slice_structured)
3125                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
3126                     break;
3127                 case AV_CODEC_ID_MPEG2VIDEO:
3128                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
3129                 case AV_CODEC_ID_MPEG1VIDEO:
3130                     if(s->mb_skip_run) is_gob_start=0;
3131                     break;
3132                 case AV_CODEC_ID_MJPEG:
3133                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
3134                     break;
3135                 }
3136
3137                 if(is_gob_start){
3138                     if(s->start_mb_y != mb_y || mb_x!=0){
3139                         write_slice_end(s);
3140
3141                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
3142                             ff_mpeg4_init_partitions(s);
3143                         }
3144                     }
3145
3146                     av_assert2((put_bits_count(&s->pb)&7) == 0);
3147                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
3148
3149                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
3150                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
3151                         int d = 100 / s->error_rate;
3152                         if(r % d == 0){
3153                             current_packet_size=0;
3154                             s->pb.buf_ptr= s->ptr_lastgob;
3155                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
3156                         }
3157                     }
3158
3159 #if FF_API_RTP_CALLBACK
3160 FF_DISABLE_DEPRECATION_WARNINGS
3161                     if (s->avctx->rtp_callback){
3162                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
3163                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
3164                     }
3165 FF_ENABLE_DEPRECATION_WARNINGS
3166 #endif
3167                     update_mb_info(s, 1);
3168
3169                     switch(s->codec_id){
3170                     case AV_CODEC_ID_MPEG4:
3171                         if (CONFIG_MPEG4_ENCODER) {
3172                             ff_mpeg4_encode_video_packet_header(s);
3173                             ff_mpeg4_clean_buffers(s);
3174                         }
3175                     break;
3176                     case AV_CODEC_ID_MPEG1VIDEO:
3177                     case AV_CODEC_ID_MPEG2VIDEO:
3178                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
3179                             ff_mpeg1_encode_slice_header(s);
3180                             ff_mpeg1_clean_buffers(s);
3181                         }
3182                     break;
3183                     case AV_CODEC_ID_H263:
3184                     case AV_CODEC_ID_H263P:
3185                         if (CONFIG_H263_ENCODER)
3186                             ff_h263_encode_gob_header(s, mb_y);
3187                     break;
3188                     }
3189
3190                     if (s->avctx->flags & AV_CODEC_FLAG_PASS1) {
3191                         int bits= put_bits_count(&s->pb);
3192                         s->misc_bits+= bits - s->last_bits;
3193                         s->last_bits= bits;
3194                     }
3195
3196                     s->ptr_lastgob += current_packet_size;
3197                     s->first_slice_line=1;
3198                     s->resync_mb_x=mb_x;
3199                     s->resync_mb_y=mb_y;
3200                 }
3201             }
3202
3203             if(  (s->resync_mb_x   == s->mb_x)
3204                && s->resync_mb_y+1 == s->mb_y){
3205                 s->first_slice_line=0;
3206             }
3207
3208             s->mb_skipped=0;
3209             s->dquant=0; //only for QP_RD
3210
3211             update_mb_info(s, 0);
3212
3213             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
3214                 int next_block=0;
3215                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
3216
3217                 copy_context_before_encode(&backup_s, s, -1);
3218                 backup_s.pb= s->pb;
3219                 best_s.data_partitioning= s->data_partitioning;
3220                 best_s.partitioned_frame= s->partitioned_frame;
3221                 if(s->data_partitioning){
3222                     backup_s.pb2= s->pb2;
3223                     backup_s.tex_pb= s->tex_pb;
3224                 }
3225
3226                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
3227                     s->mv_dir = MV_DIR_FORWARD;
3228                     s->mv_type = MV_TYPE_16X16;
3229                     s->mb_intra= 0;
3230                     s->mv[0][0][0] = s->p_mv_table[xy][0];
3231                     s->mv[0][0][1] = s->p_mv_table[xy][1];
3232                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
3233                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
3234                 }
3235                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
3236                     s->mv_dir = MV_DIR_FORWARD;
3237                     s->mv_type = MV_TYPE_FIELD;
3238                     s->mb_intra= 0;
3239                     for(i=0; i<2; i++){
3240                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3241                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3242                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3243                     }
3244                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
3245                                  &dmin, &next_block, 0, 0);
3246                 }
3247                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
3248                     s->mv_dir = MV_DIR_FORWARD;
3249                     s->mv_type = MV_TYPE_16X16;
3250                     s->mb_intra= 0;
3251                     s->mv[0][0][0] = 0;
3252                     s->mv[0][0][1] = 0;
3253                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
3254                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
3255                 }
3256                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
3257                     s->mv_dir = MV_DIR_FORWARD;
3258                     s->mv_type = MV_TYPE_8X8;
3259                     s->mb_intra= 0;
3260                     for(i=0; i<4; i++){
3261                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3262                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3263                     }
3264                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
3265                                  &dmin, &next_block, 0, 0);
3266                 }
3267                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
3268                     s->mv_dir = MV_DIR_FORWARD;
3269                     s->mv_type = MV_TYPE_16X16;
3270                     s->mb_intra= 0;
3271                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3272                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3273                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
3274                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
3275                 }
3276                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
3277                     s->mv_dir = MV_DIR_BACKWARD;
3278                     s->mv_type = MV_TYPE_16X16;
3279                     s->mb_intra= 0;
3280                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3281                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3282                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
3283                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
3284                 }
3285                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
3286                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3287                     s->mv_type = MV_TYPE_16X16;
3288                     s->mb_intra= 0;
3289                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3290                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3291                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3292                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3293                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
3294                                  &dmin, &next_block, 0, 0);
3295                 }
3296                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
3297                     s->mv_dir = MV_DIR_FORWARD;
3298                     s->mv_type = MV_TYPE_FIELD;
3299                     s->mb_intra= 0;
3300                     for(i=0; i<2; i++){
3301                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3302                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3303                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3304                     }
3305                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
3306                                  &dmin, &next_block, 0, 0);
3307                 }
3308                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
3309                     s->mv_dir = MV_DIR_BACKWARD;
3310                     s->mv_type = MV_TYPE_FIELD;
3311                     s->mb_intra= 0;
3312                     for(i=0; i<2; i++){
3313                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3314                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3315                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3316                     }
3317                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
3318                                  &dmin, &next_block, 0, 0);
3319                 }
3320                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
3321                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3322                     s->mv_type = MV_TYPE_FIELD;
3323                     s->mb_intra= 0;
3324                     for(dir=0; dir<2; dir++){
3325                         for(i=0; i<2; i++){
3326                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3327                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3328                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3329                         }
3330                     }
3331                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
3332                                  &dmin, &next_block, 0, 0);
3333                 }
3334                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
3335                     s->mv_dir = 0;
3336                     s->mv_type = MV_TYPE_16X16;
3337                     s->mb_intra= 1;
3338                     s->mv[0][0][0] = 0;
3339                     s->mv[0][0][1] = 0;
3340                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
3341                                  &dmin, &next_block, 0, 0);
3342                     if(s->h263_pred || s->h263_aic){
3343                         if(best_s.mb_intra)
3344                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
3345                         else
3346                             ff_clean_intra_table_entries(s); //old mode?
3347                     }
3348                 }
3349
3350                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
3351                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
3352                         const int last_qp= backup_s.qscale;
3353                         int qpi, qp, dc[6];
3354                         int16_t ac[6][16];
3355                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
3356                         static const int dquant_tab[4]={-1,1,-2,2};
3357                         int storecoefs = s->mb_intra && s->dc_val[0];
3358
3359                         av_assert2(backup_s.dquant == 0);
3360
3361                         //FIXME intra
3362                         s->mv_dir= best_s.mv_dir;
3363                         s->mv_type = MV_TYPE_16X16;
3364 &n