fate: add more tests for hue video filter
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of FFmpeg.
9  *
10  * FFmpeg is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * FFmpeg is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with FFmpeg; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /*
26  * non linear quantizers with large QPs and VBV with restrictive qmin fixes sponsored by NOA GmbH
27  */
28
29 /**
30  * @file
31  * The simplest mpeg encoder (well, it was the simplest!).
32  */
33
34 #include <stdint.h>
35
36 #include "libavutil/internal.h"
37 #include "libavutil/intmath.h"
38 #include "libavutil/mathematics.h"
39 #include "libavutil/pixdesc.h"
40 #include "libavutil/opt.h"
41 #include "libavutil/timer.h"
42 #include "avcodec.h"
43 #include "dct.h"
44 #include "idctdsp.h"
45 #include "mpeg12.h"
46 #include "mpegvideo.h"
47 #include "mpegvideodata.h"
48 #include "h261.h"
49 #include "h263.h"
50 #include "h263data.h"
51 #include "mjpegenc_common.h"
52 #include "mathops.h"
53 #include "mpegutils.h"
54 #include "mjpegenc.h"
55 #include "msmpeg4.h"
56 #include "pixblockdsp.h"
57 #include "qpeldsp.h"
58 #include "faandct.h"
59 #include "thread.h"
60 #include "aandcttab.h"
61 #include "flv.h"
62 #include "mpeg4video.h"
63 #include "internal.h"
64 #include "bytestream.h"
65 #include "wmv2.h"
66 #include "rv10.h"
67 #include "libxvid.h"
68 #include <limits.h>
69 #include "sp5x.h"
70
71 #define QUANT_BIAS_SHIFT 8
72
73 #define QMAT_SHIFT_MMX 16
74 #define QMAT_SHIFT 21
75
76 static int encode_picture(MpegEncContext *s, int picture_number);
77 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
78 static int sse_mb(MpegEncContext *s);
79 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
80 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
81
82 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
83 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
84
85 const AVOption ff_mpv_generic_options[] = {
86     FF_MPV_COMMON_OPTS
87     { NULL },
88 };
89
90 void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64],
91                        uint16_t (*qmat16)[2][64],
92                        const uint16_t *quant_matrix,
93                        int bias, int qmin, int qmax, int intra)
94 {
95     FDCTDSPContext *fdsp = &s->fdsp;
96     int qscale;
97     int shift = 0;
98
99     for (qscale = qmin; qscale <= qmax; qscale++) {
100         int i;
101         int qscale2;
102
103         if (s->q_scale_type) qscale2 = ff_mpeg2_non_linear_qscale[qscale];
104         else                 qscale2 = qscale << 1;
105
106         if (fdsp->fdct == ff_jpeg_fdct_islow_8  ||
107 #if CONFIG_FAANDCT
108             fdsp->fdct == ff_faandct            ||
109 #endif /* CONFIG_FAANDCT */
110             fdsp->fdct == ff_jpeg_fdct_islow_10) {
111             for (i = 0; i < 64; i++) {
112                 const int j = s->idsp.idct_permutation[i];
113                 int64_t den = (int64_t) qscale2 * quant_matrix[j];
114                 /* 16 <= qscale * quant_matrix[i] <= 7905
115                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
116                  *             19952 <=              x  <= 249205026
117                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
118                  *           3444240 >= (1 << 36) / (x) >= 275 */
119
120                 qmat[qscale][i] = (int)((UINT64_C(2) << QMAT_SHIFT) / den);
121             }
122         } else if (fdsp->fdct == ff_fdct_ifast) {
123             for (i = 0; i < 64; i++) {
124                 const int j = s->idsp.idct_permutation[i];
125                 int64_t den = ff_aanscales[i] * (int64_t) qscale2 * quant_matrix[j];
126                 /* 16 <= qscale * quant_matrix[i] <= 7905
127                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
128                  *             19952 <=              x  <= 249205026
129                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
130                  *           3444240 >= (1 << 36) / (x) >= 275 */
131
132                 qmat[qscale][i] = (int)((UINT64_C(2) << (QMAT_SHIFT + 14)) / den);
133             }
134         } else {
135             for (i = 0; i < 64; i++) {
136                 const int j = s->idsp.idct_permutation[i];
137                 int64_t den = (int64_t) qscale2 * quant_matrix[j];
138                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
139                  * Assume x = qscale * quant_matrix[i]
140                  * So             16 <=              x  <= 7905
141                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
142                  * so          32768 >= (1 << 19) / (x) >= 67 */
143                 qmat[qscale][i] = (int)((UINT64_C(2) << QMAT_SHIFT) / den);
144                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
145                 //                    (qscale * quant_matrix[i]);
146                 qmat16[qscale][0][i] = (2 << QMAT_SHIFT_MMX) / den;
147
148                 if (qmat16[qscale][0][i] == 0 ||
149                     qmat16[qscale][0][i] == 128 * 256)
150                     qmat16[qscale][0][i] = 128 * 256 - 1;
151                 qmat16[qscale][1][i] =
152                     ROUNDED_DIV(bias * (1<<(16 - QUANT_BIAS_SHIFT)),
153                                 qmat16[qscale][0][i]);
154             }
155         }
156
157         for (i = intra; i < 64; i++) {
158             int64_t max = 8191;
159             if (fdsp->fdct == ff_fdct_ifast) {
160                 max = (8191LL * ff_aanscales[i]) >> 14;
161             }
162             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
163                 shift++;
164             }
165         }
166     }
167     if (shift) {
168         av_log(NULL, AV_LOG_INFO,
169                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
170                QMAT_SHIFT - shift);
171     }
172 }
173
174 static inline void update_qscale(MpegEncContext *s)
175 {
176     if (s->q_scale_type == 1 && 0) {
177         int i;
178         int bestdiff=INT_MAX;
179         int best = 1;
180
181         for (i = 0 ; i<FF_ARRAY_ELEMS(ff_mpeg2_non_linear_qscale); i++) {
182             int diff = FFABS((ff_mpeg2_non_linear_qscale[i]<<(FF_LAMBDA_SHIFT + 6)) - (int)s->lambda * 139);
183             if (ff_mpeg2_non_linear_qscale[i] < s->avctx->qmin ||
184                 (ff_mpeg2_non_linear_qscale[i] > s->avctx->qmax && !s->vbv_ignore_qmax))
185                 continue;
186             if (diff < bestdiff) {
187                 bestdiff = diff;
188                 best = i;
189             }
190         }
191         s->qscale = best;
192     } else {
193         s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
194                     (FF_LAMBDA_SHIFT + 7);
195         s->qscale = av_clip(s->qscale, s->avctx->qmin, s->vbv_ignore_qmax ? 31 : s->avctx->qmax);
196     }
197
198     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
199                  FF_LAMBDA_SHIFT;
200 }
201
202 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
203 {
204     int i;
205
206     if (matrix) {
207         put_bits(pb, 1, 1);
208         for (i = 0; i < 64; i++) {
209             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
210         }
211     } else
212         put_bits(pb, 1, 0);
213 }
214
215 /**
216  * init s->current_picture.qscale_table from s->lambda_table
217  */
218 void ff_init_qscale_tab(MpegEncContext *s)
219 {
220     int8_t * const qscale_table = s->current_picture.qscale_table;
221     int i;
222
223     for (i = 0; i < s->mb_num; i++) {
224         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
225         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
226         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
227                                                   s->avctx->qmax);
228     }
229 }
230
231 static void update_duplicate_context_after_me(MpegEncContext *dst,
232                                               MpegEncContext *src)
233 {
234 #define COPY(a) dst->a= src->a
235     COPY(pict_type);
236     COPY(current_picture);
237     COPY(f_code);
238     COPY(b_code);
239     COPY(qscale);
240     COPY(lambda);
241     COPY(lambda2);
242     COPY(picture_in_gop_number);
243     COPY(gop_picture_number);
244     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
245     COPY(progressive_frame);    // FIXME don't set in encode_header
246     COPY(partitioned_frame);    // FIXME don't set in encode_header
247 #undef COPY
248 }
249
250 /**
251  * Set the given MpegEncContext to defaults for encoding.
252  * the changed fields will not depend upon the prior state of the MpegEncContext.
253  */
254 static void mpv_encode_defaults(MpegEncContext *s)
255 {
256     int i;
257     ff_mpv_common_defaults(s);
258
259     for (i = -16; i < 16; i++) {
260         default_fcode_tab[i + MAX_MV] = 1;
261     }
262     s->me.mv_penalty = default_mv_penalty;
263     s->fcode_tab     = default_fcode_tab;
264
265     s->input_picture_number  = 0;
266     s->picture_in_gop_number = 0;
267 }
268
269 av_cold int ff_dct_encode_init(MpegEncContext *s)
270 {
271     if (ARCH_X86)
272         ff_dct_encode_init_x86(s);
273
274     if (CONFIG_H263_ENCODER)
275         ff_h263dsp_init(&s->h263dsp);
276     if (!s->dct_quantize)
277         s->dct_quantize = ff_dct_quantize_c;
278     if (!s->denoise_dct)
279         s->denoise_dct  = denoise_dct_c;
280     s->fast_dct_quantize = s->dct_quantize;
281     if (s->avctx->trellis)
282         s->dct_quantize  = dct_quantize_trellis_c;
283
284     return 0;
285 }
286
287 /* init video encoder */
288 av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
289 {
290     MpegEncContext *s = avctx->priv_data;
291     AVCPBProperties *cpb_props;
292     int i, ret, format_supported;
293
294     mpv_encode_defaults(s);
295
296     switch (avctx->codec_id) {
297     case AV_CODEC_ID_MPEG2VIDEO:
298         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
299             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
300             av_log(avctx, AV_LOG_ERROR,
301                    "only YUV420 and YUV422 are supported\n");
302             return -1;
303         }
304         break;
305     case AV_CODEC_ID_MJPEG:
306     case AV_CODEC_ID_AMV:
307         format_supported = 0;
308         /* JPEG color space */
309         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
310             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
311             avctx->pix_fmt == AV_PIX_FMT_YUVJ444P ||
312             (avctx->color_range == AVCOL_RANGE_JPEG &&
313              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
314               avctx->pix_fmt == AV_PIX_FMT_YUV422P ||
315               avctx->pix_fmt == AV_PIX_FMT_YUV444P)))
316             format_supported = 1;
317         /* MPEG color space */
318         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
319                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
320                   avctx->pix_fmt == AV_PIX_FMT_YUV422P ||
321                   avctx->pix_fmt == AV_PIX_FMT_YUV444P))
322             format_supported = 1;
323
324         if (!format_supported) {
325             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
326             return -1;
327         }
328         break;
329     default:
330         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
331             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
332             return -1;
333         }
334     }
335
336     switch (avctx->pix_fmt) {
337     case AV_PIX_FMT_YUVJ444P:
338     case AV_PIX_FMT_YUV444P:
339         s->chroma_format = CHROMA_444;
340         break;
341     case AV_PIX_FMT_YUVJ422P:
342     case AV_PIX_FMT_YUV422P:
343         s->chroma_format = CHROMA_422;
344         break;
345     case AV_PIX_FMT_YUVJ420P:
346     case AV_PIX_FMT_YUV420P:
347     default:
348         s->chroma_format = CHROMA_420;
349         break;
350     }
351
352     avctx->bits_per_raw_sample = av_clip(avctx->bits_per_raw_sample, 0, 8);
353
354 #if FF_API_PRIVATE_OPT
355 FF_DISABLE_DEPRECATION_WARNINGS
356     if (avctx->rtp_payload_size)
357         s->rtp_payload_size = avctx->rtp_payload_size;
358     if (avctx->me_penalty_compensation)
359         s->me_penalty_compensation = avctx->me_penalty_compensation;
360     if (avctx->pre_me)
361         s->me_pre = avctx->pre_me;
362 FF_ENABLE_DEPRECATION_WARNINGS
363 #endif
364
365     s->bit_rate = avctx->bit_rate;
366     s->width    = avctx->width;
367     s->height   = avctx->height;
368     if (avctx->gop_size > 600 &&
369         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
370         av_log(avctx, AV_LOG_WARNING,
371                "keyframe interval too large!, reducing it from %d to %d\n",
372                avctx->gop_size, 600);
373         avctx->gop_size = 600;
374     }
375     s->gop_size     = avctx->gop_size;
376     s->avctx        = avctx;
377     if (avctx->max_b_frames > MAX_B_FRAMES) {
378         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
379                "is %d.\n", MAX_B_FRAMES);
380         avctx->max_b_frames = MAX_B_FRAMES;
381     }
382     s->max_b_frames = avctx->max_b_frames;
383     s->codec_id     = avctx->codec->id;
384     s->strict_std_compliance = avctx->strict_std_compliance;
385     s->quarter_sample     = (avctx->flags & AV_CODEC_FLAG_QPEL) != 0;
386     s->rtp_mode           = !!s->rtp_payload_size;
387     s->intra_dc_precision = avctx->intra_dc_precision;
388
389     // workaround some differences between how applications specify dc precision
390     if (s->intra_dc_precision < 0) {
391         s->intra_dc_precision += 8;
392     } else if (s->intra_dc_precision >= 8)
393         s->intra_dc_precision -= 8;
394
395     if (s->intra_dc_precision < 0) {
396         av_log(avctx, AV_LOG_ERROR,
397                 "intra dc precision must be positive, note some applications use"
398                 " 0 and some 8 as base meaning 8bit, the value must not be smaller than that\n");
399         return AVERROR(EINVAL);
400     }
401
402     if (avctx->codec_id == AV_CODEC_ID_AMV || (avctx->active_thread_type & FF_THREAD_SLICE))
403         s->huffman = 0;
404
405     if (s->intra_dc_precision > (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO ? 3 : 0)) {
406         av_log(avctx, AV_LOG_ERROR, "intra dc precision too large\n");
407         return AVERROR(EINVAL);
408     }
409     s->user_specified_pts = AV_NOPTS_VALUE;
410
411     if (s->gop_size <= 1) {
412         s->intra_only = 1;
413         s->gop_size   = 12;
414     } else {
415         s->intra_only = 0;
416     }
417
418     /* Fixed QSCALE */
419     s->fixed_qscale = !!(avctx->flags & AV_CODEC_FLAG_QSCALE);
420
421     s->adaptive_quant = (s->avctx->lumi_masking ||
422                          s->avctx->dark_masking ||
423                          s->avctx->temporal_cplx_masking ||
424                          s->avctx->spatial_cplx_masking  ||
425                          s->avctx->p_masking      ||
426                          s->border_masking ||
427                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
428                         !s->fixed_qscale;
429
430     s->loop_filter = !!(s->avctx->flags & AV_CODEC_FLAG_LOOP_FILTER);
431
432     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
433         switch(avctx->codec_id) {
434         case AV_CODEC_ID_MPEG1VIDEO:
435         case AV_CODEC_ID_MPEG2VIDEO:
436             avctx->rc_buffer_size = FFMAX(avctx->rc_max_rate, 15000000) * 112LL / 15000000 * 16384;
437             break;
438         case AV_CODEC_ID_MPEG4:
439         case AV_CODEC_ID_MSMPEG4V1:
440         case AV_CODEC_ID_MSMPEG4V2:
441         case AV_CODEC_ID_MSMPEG4V3:
442             if       (avctx->rc_max_rate >= 15000000) {
443                 avctx->rc_buffer_size = 320 + (avctx->rc_max_rate - 15000000LL) * (760-320) / (38400000 - 15000000);
444             } else if(avctx->rc_max_rate >=  2000000) {
445                 avctx->rc_buffer_size =  80 + (avctx->rc_max_rate -  2000000LL) * (320- 80) / (15000000 -  2000000);
446             } else if(avctx->rc_max_rate >=   384000) {
447                 avctx->rc_buffer_size =  40 + (avctx->rc_max_rate -   384000LL) * ( 80- 40) / ( 2000000 -   384000);
448             } else
449                 avctx->rc_buffer_size = 40;
450             avctx->rc_buffer_size *= 16384;
451             break;
452         }
453         if (avctx->rc_buffer_size) {
454             av_log(avctx, AV_LOG_INFO, "Automatically choosing VBV buffer size of %d kbyte\n", avctx->rc_buffer_size/8192);
455         }
456     }
457
458     if ((!avctx->rc_max_rate) != (!avctx->rc_buffer_size)) {
459         av_log(avctx, AV_LOG_ERROR, "Either both buffer size and max rate or neither must be specified\n");
460         return -1;
461     }
462
463     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
464         av_log(avctx, AV_LOG_INFO,
465                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
466     }
467
468     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
469         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
470         return -1;
471     }
472
473     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
474         av_log(avctx, AV_LOG_ERROR, "bitrate above max bitrate\n");
475         return -1;
476     }
477
478     if (avctx->rc_max_rate &&
479         avctx->rc_max_rate == avctx->bit_rate &&
480         avctx->rc_max_rate != avctx->rc_min_rate) {
481         av_log(avctx, AV_LOG_INFO,
482                "impossible bitrate constraints, this will fail\n");
483     }
484
485     if (avctx->rc_buffer_size &&
486         avctx->bit_rate * (int64_t)avctx->time_base.num >
487             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
488         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
489         return -1;
490     }
491
492     if (!s->fixed_qscale &&
493         avctx->bit_rate * av_q2d(avctx->time_base) >
494             avctx->bit_rate_tolerance) {
495         av_log(avctx, AV_LOG_WARNING,
496                "bitrate tolerance %d too small for bitrate %"PRId64", overriding\n", avctx->bit_rate_tolerance, avctx->bit_rate);
497         avctx->bit_rate_tolerance = 5 * avctx->bit_rate * av_q2d(avctx->time_base);
498     }
499
500     if (s->avctx->rc_max_rate &&
501         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
502         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
503          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
504         90000LL * (avctx->rc_buffer_size - 1) >
505             s->avctx->rc_max_rate * 0xFFFFLL) {
506         av_log(avctx, AV_LOG_INFO,
507                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
508                "specified vbv buffer is too large for the given bitrate!\n");
509     }
510
511     if ((s->avctx->flags & AV_CODEC_FLAG_4MV) && s->codec_id != AV_CODEC_ID_MPEG4 &&
512         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
513         s->codec_id != AV_CODEC_ID_FLV1) {
514         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
515         return -1;
516     }
517
518     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
519         av_log(avctx, AV_LOG_ERROR,
520                "OBMC is only supported with simple mb decision\n");
521         return -1;
522     }
523
524     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
525         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
526         return -1;
527     }
528
529     if (s->max_b_frames                    &&
530         s->codec_id != AV_CODEC_ID_MPEG4      &&
531         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
532         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
533         av_log(avctx, AV_LOG_ERROR, "B-frames not supported by codec\n");
534         return -1;
535     }
536     if (s->max_b_frames < 0) {
537         av_log(avctx, AV_LOG_ERROR,
538                "max b frames must be 0 or positive for mpegvideo based encoders\n");
539         return -1;
540     }
541
542     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
543          s->codec_id == AV_CODEC_ID_H263  ||
544          s->codec_id == AV_CODEC_ID_H263P) &&
545         (avctx->sample_aspect_ratio.num > 255 ||
546          avctx->sample_aspect_ratio.den > 255)) {
547         av_log(avctx, AV_LOG_WARNING,
548                "Invalid pixel aspect ratio %i/%i, limit is 255/255 reducing\n",
549                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
550         av_reduce(&avctx->sample_aspect_ratio.num, &avctx->sample_aspect_ratio.den,
551                    avctx->sample_aspect_ratio.num,  avctx->sample_aspect_ratio.den, 255);
552     }
553
554     if ((s->codec_id == AV_CODEC_ID_H263  ||
555          s->codec_id == AV_CODEC_ID_H263P) &&
556         (avctx->width  > 2048 ||
557          avctx->height > 1152 )) {
558         av_log(avctx, AV_LOG_ERROR, "H.263 does not support resolutions above 2048x1152\n");
559         return -1;
560     }
561     if ((s->codec_id == AV_CODEC_ID_H263  ||
562          s->codec_id == AV_CODEC_ID_H263P) &&
563         ((avctx->width &3) ||
564          (avctx->height&3) )) {
565         av_log(avctx, AV_LOG_ERROR, "w/h must be a multiple of 4\n");
566         return -1;
567     }
568
569     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO &&
570         (avctx->width  > 4095 ||
571          avctx->height > 4095 )) {
572         av_log(avctx, AV_LOG_ERROR, "MPEG-1 does not support resolutions above 4095x4095\n");
573         return -1;
574     }
575
576     if (s->codec_id == AV_CODEC_ID_MPEG2VIDEO &&
577         (avctx->width  > 16383 ||
578          avctx->height > 16383 )) {
579         av_log(avctx, AV_LOG_ERROR, "MPEG-2 does not support resolutions above 16383x16383\n");
580         return -1;
581     }
582
583     if (s->codec_id == AV_CODEC_ID_RV10 &&
584         (avctx->width &15 ||
585          avctx->height&15 )) {
586         av_log(avctx, AV_LOG_ERROR, "width and height must be a multiple of 16\n");
587         return AVERROR(EINVAL);
588     }
589
590     if (s->codec_id == AV_CODEC_ID_RV20 &&
591         (avctx->width &3 ||
592          avctx->height&3 )) {
593         av_log(avctx, AV_LOG_ERROR, "width and height must be a multiple of 4\n");
594         return AVERROR(EINVAL);
595     }
596
597     if ((s->codec_id == AV_CODEC_ID_WMV1 ||
598          s->codec_id == AV_CODEC_ID_WMV2) &&
599          avctx->width & 1) {
600          av_log(avctx, AV_LOG_ERROR, "width must be multiple of 2\n");
601          return -1;
602     }
603
604     if ((s->avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT | AV_CODEC_FLAG_INTERLACED_ME)) &&
605         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
606         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
607         return -1;
608     }
609
610 #if FF_API_PRIVATE_OPT
611     FF_DISABLE_DEPRECATION_WARNINGS
612     if (avctx->mpeg_quant)
613         s->mpeg_quant = avctx->mpeg_quant;
614     FF_ENABLE_DEPRECATION_WARNINGS
615 #endif
616
617     // FIXME mpeg2 uses that too
618     if (s->mpeg_quant && (   s->codec_id != AV_CODEC_ID_MPEG4
619                           && s->codec_id != AV_CODEC_ID_MPEG2VIDEO)) {
620         av_log(avctx, AV_LOG_ERROR,
621                "mpeg2 style quantization not supported by codec\n");
622         return -1;
623     }
624
625     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
626         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
627         return -1;
628     }
629
630     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
631         s->avctx->mb_decision != FF_MB_DECISION_RD) {
632         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
633         return -1;
634     }
635
636     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
637             (s->codec_id == AV_CODEC_ID_AMV ||
638              s->codec_id == AV_CODEC_ID_MJPEG)) {
639         // Used to produce garbage with MJPEG.
640         av_log(avctx, AV_LOG_ERROR,
641                "QP RD is no longer compatible with MJPEG or AMV\n");
642         return -1;
643     }
644
645 #if FF_API_PRIVATE_OPT
646 FF_DISABLE_DEPRECATION_WARNINGS
647     if (avctx->scenechange_threshold)
648         s->scenechange_threshold = avctx->scenechange_threshold;
649 FF_ENABLE_DEPRECATION_WARNINGS
650 #endif
651
652     if (s->scenechange_threshold < 1000000000 &&
653         (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)) {
654         av_log(avctx, AV_LOG_ERROR,
655                "closed gop with scene change detection are not supported yet, "
656                "set threshold to 1000000000\n");
657         return -1;
658     }
659
660     if (s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY) {
661         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
662             s->strict_std_compliance >= FF_COMPLIANCE_NORMAL) {
663             av_log(avctx, AV_LOG_ERROR,
664                    "low delay forcing is only available for mpeg2, "
665                    "set strict_std_compliance to 'unofficial' or lower in order to allow it\n");
666             return -1;
667         }
668         if (s->max_b_frames != 0) {
669             av_log(avctx, AV_LOG_ERROR,
670                    "B-frames cannot be used with low delay\n");
671             return -1;
672         }
673     }
674
675     if (s->q_scale_type == 1) {
676         if (avctx->qmax > 28) {
677             av_log(avctx, AV_LOG_ERROR,
678                    "non linear quant only supports qmax <= 28 currently\n");
679             return -1;
680         }
681     }
682
683     if (avctx->slices > 1 &&
684         (avctx->codec_id == AV_CODEC_ID_FLV1 || avctx->codec_id == AV_CODEC_ID_H261)) {
685         av_log(avctx, AV_LOG_ERROR, "Multiple slices are not supported by this codec\n");
686         return AVERROR(EINVAL);
687     }
688
689     if (s->avctx->thread_count > 1         &&
690         s->codec_id != AV_CODEC_ID_MPEG4      &&
691         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
692         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
693         s->codec_id != AV_CODEC_ID_MJPEG      &&
694         (s->codec_id != AV_CODEC_ID_H263P)) {
695         av_log(avctx, AV_LOG_ERROR,
696                "multi threaded encoding not supported by codec\n");
697         return -1;
698     }
699
700     if (s->avctx->thread_count < 1) {
701         av_log(avctx, AV_LOG_ERROR,
702                "automatic thread number detection not supported by codec, "
703                "patch welcome\n");
704         return -1;
705     }
706
707     if (!avctx->time_base.den || !avctx->time_base.num) {
708         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
709         return -1;
710     }
711
712 #if FF_API_PRIVATE_OPT
713 FF_DISABLE_DEPRECATION_WARNINGS
714     if (avctx->b_frame_strategy)
715         s->b_frame_strategy = avctx->b_frame_strategy;
716     if (avctx->b_sensitivity != 40)
717         s->b_sensitivity = avctx->b_sensitivity;
718 FF_ENABLE_DEPRECATION_WARNINGS
719 #endif
720
721     if (s->b_frame_strategy && (avctx->flags & AV_CODEC_FLAG_PASS2)) {
722         av_log(avctx, AV_LOG_INFO,
723                "notice: b_frame_strategy only affects the first pass\n");
724         s->b_frame_strategy = 0;
725     }
726
727     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
728     if (i > 1) {
729         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
730         avctx->time_base.den /= i;
731         avctx->time_base.num /= i;
732         //return -1;
733     }
734
735     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG || s->codec_id==AV_CODEC_ID_AMV) {
736         // (a + x * 3 / 8) / x
737         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
738         s->inter_quant_bias = 0;
739     } else {
740         s->intra_quant_bias = 0;
741         // (a - x / 4) / x
742         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
743     }
744
745     if (avctx->qmin > avctx->qmax || avctx->qmin <= 0) {
746         av_log(avctx, AV_LOG_ERROR, "qmin and or qmax are invalid, they must be 0 < min <= max\n");
747         return AVERROR(EINVAL);
748     }
749
750     av_log(avctx, AV_LOG_DEBUG, "intra_quant_bias = %d inter_quant_bias = %d\n",s->intra_quant_bias,s->inter_quant_bias);
751
752     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
753         s->avctx->time_base.den > (1 << 16) - 1) {
754         av_log(avctx, AV_LOG_ERROR,
755                "timebase %d/%d not supported by MPEG 4 standard, "
756                "the maximum admitted value for the timebase denominator "
757                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
758                (1 << 16) - 1);
759         return -1;
760     }
761     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
762
763     switch (avctx->codec->id) {
764     case AV_CODEC_ID_MPEG1VIDEO:
765         s->out_format = FMT_MPEG1;
766         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
767         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
768         break;
769     case AV_CODEC_ID_MPEG2VIDEO:
770         s->out_format = FMT_MPEG1;
771         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
772         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
773         s->rtp_mode   = 1;
774         break;
775     case AV_CODEC_ID_MJPEG:
776     case AV_CODEC_ID_AMV:
777         s->out_format = FMT_MJPEG;
778         s->intra_only = 1; /* force intra only for jpeg */
779         if (!CONFIG_MJPEG_ENCODER ||
780             ff_mjpeg_encode_init(s) < 0)
781             return -1;
782         avctx->delay = 0;
783         s->low_delay = 1;
784         break;
785     case AV_CODEC_ID_H261:
786         if (!CONFIG_H261_ENCODER)
787             return -1;
788         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
789             av_log(avctx, AV_LOG_ERROR,
790                    "The specified picture size of %dx%d is not valid for the "
791                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
792                     s->width, s->height);
793             return -1;
794         }
795         s->out_format = FMT_H261;
796         avctx->delay  = 0;
797         s->low_delay  = 1;
798         s->rtp_mode   = 0; /* Sliced encoding not supported */
799         break;
800     case AV_CODEC_ID_H263:
801         if (!CONFIG_H263_ENCODER)
802             return -1;
803         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
804                              s->width, s->height) == 8) {
805             av_log(avctx, AV_LOG_ERROR,
806                    "The specified picture size of %dx%d is not valid for "
807                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
808                    "352x288, 704x576, and 1408x1152. "
809                    "Try H.263+.\n", s->width, s->height);
810             return -1;
811         }
812         s->out_format = FMT_H263;
813         avctx->delay  = 0;
814         s->low_delay  = 1;
815         break;
816     case AV_CODEC_ID_H263P:
817         s->out_format = FMT_H263;
818         s->h263_plus  = 1;
819         /* Fx */
820         s->h263_aic        = (avctx->flags & AV_CODEC_FLAG_AC_PRED) ? 1 : 0;
821         s->modified_quant  = s->h263_aic;
822         s->loop_filter     = (avctx->flags & AV_CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
823         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
824
825         /* /Fx */
826         /* These are just to be sure */
827         avctx->delay = 0;
828         s->low_delay = 1;
829         break;
830     case AV_CODEC_ID_FLV1:
831         s->out_format      = FMT_H263;
832         s->h263_flv        = 2; /* format = 1; 11-bit codes */
833         s->unrestricted_mv = 1;
834         s->rtp_mode  = 0; /* don't allow GOB */
835         avctx->delay = 0;
836         s->low_delay = 1;
837         break;
838     case AV_CODEC_ID_RV10:
839         s->out_format = FMT_H263;
840         avctx->delay  = 0;
841         s->low_delay  = 1;
842         break;
843     case AV_CODEC_ID_RV20:
844         s->out_format      = FMT_H263;
845         avctx->delay       = 0;
846         s->low_delay       = 1;
847         s->modified_quant  = 1;
848         s->h263_aic        = 1;
849         s->h263_plus       = 1;
850         s->loop_filter     = 1;
851         s->unrestricted_mv = 0;
852         break;
853     case AV_CODEC_ID_MPEG4:
854         s->out_format      = FMT_H263;
855         s->h263_pred       = 1;
856         s->unrestricted_mv = 1;
857         s->low_delay       = s->max_b_frames ? 0 : 1;
858         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
859         break;
860     case AV_CODEC_ID_MSMPEG4V2:
861         s->out_format      = FMT_H263;
862         s->h263_pred       = 1;
863         s->unrestricted_mv = 1;
864         s->msmpeg4_version = 2;
865         avctx->delay       = 0;
866         s->low_delay       = 1;
867         break;
868     case AV_CODEC_ID_MSMPEG4V3:
869         s->out_format        = FMT_H263;
870         s->h263_pred         = 1;
871         s->unrestricted_mv   = 1;
872         s->msmpeg4_version   = 3;
873         s->flipflop_rounding = 1;
874         avctx->delay         = 0;
875         s->low_delay         = 1;
876         break;
877     case AV_CODEC_ID_WMV1:
878         s->out_format        = FMT_H263;
879         s->h263_pred         = 1;
880         s->unrestricted_mv   = 1;
881         s->msmpeg4_version   = 4;
882         s->flipflop_rounding = 1;
883         avctx->delay         = 0;
884         s->low_delay         = 1;
885         break;
886     case AV_CODEC_ID_WMV2:
887         s->out_format        = FMT_H263;
888         s->h263_pred         = 1;
889         s->unrestricted_mv   = 1;
890         s->msmpeg4_version   = 5;
891         s->flipflop_rounding = 1;
892         avctx->delay         = 0;
893         s->low_delay         = 1;
894         break;
895     default:
896         return -1;
897     }
898
899 #if FF_API_PRIVATE_OPT
900     FF_DISABLE_DEPRECATION_WARNINGS
901     if (avctx->noise_reduction)
902         s->noise_reduction = avctx->noise_reduction;
903     FF_ENABLE_DEPRECATION_WARNINGS
904 #endif
905
906     avctx->has_b_frames = !s->low_delay;
907
908     s->encoding = 1;
909
910     s->progressive_frame    =
911     s->progressive_sequence = !(avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT |
912                                                 AV_CODEC_FLAG_INTERLACED_ME) ||
913                                 s->alternate_scan);
914
915     /* init */
916     ff_mpv_idct_init(s);
917     if (ff_mpv_common_init(s) < 0)
918         return -1;
919
920     ff_fdctdsp_init(&s->fdsp, avctx);
921     ff_me_cmp_init(&s->mecc, avctx);
922     ff_mpegvideoencdsp_init(&s->mpvencdsp, avctx);
923     ff_pixblockdsp_init(&s->pdsp, avctx);
924     ff_qpeldsp_init(&s->qdsp);
925
926     if (s->msmpeg4_version) {
927         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
928                           2 * 2 * (MAX_LEVEL + 1) *
929                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
930     }
931     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
932
933     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
934     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_chroma_intra_matrix, 64 * 32 * sizeof(int), fail);
935     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
936     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
937     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_chroma_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
938     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
939     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
940                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
941     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
942                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
943
944
945     if (s->noise_reduction) {
946         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
947                           2 * 64 * sizeof(uint16_t), fail);
948     }
949
950     ff_dct_encode_init(s);
951
952     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
953         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
954
955     if (s->slice_context_count > 1) {
956         s->rtp_mode = 1;
957
958         if (avctx->codec_id == AV_CODEC_ID_H263P)
959             s->h263_slice_structured = 1;
960     }
961
962     s->quant_precision = 5;
963
964 #if FF_API_PRIVATE_OPT
965 FF_DISABLE_DEPRECATION_WARNINGS
966     if (avctx->frame_skip_threshold)
967         s->frame_skip_threshold = avctx->frame_skip_threshold;
968     if (avctx->frame_skip_factor)
969         s->frame_skip_factor = avctx->frame_skip_factor;
970     if (avctx->frame_skip_exp)
971         s->frame_skip_exp = avctx->frame_skip_exp;
972     if (avctx->frame_skip_cmp != FF_CMP_DCTMAX)
973         s->frame_skip_cmp = avctx->frame_skip_cmp;
974 FF_ENABLE_DEPRECATION_WARNINGS
975 #endif
976
977     ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      s->avctx->ildct_cmp);
978     ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp);
979
980     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
981         ff_h261_encode_init(s);
982     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
983         ff_h263_encode_init(s);
984     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
985         if ((ret = ff_msmpeg4_encode_init(s)) < 0)
986             return ret;
987     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
988         && s->out_format == FMT_MPEG1)
989         ff_mpeg1_encode_init(s);
990
991     /* init q matrix */
992     for (i = 0; i < 64; i++) {
993         int j = s->idsp.idct_permutation[i];
994         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
995             s->mpeg_quant) {
996             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
997             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
998         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
999             s->intra_matrix[j] =
1000             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
1001         } else {
1002             /* MPEG-1/2 */
1003             s->chroma_intra_matrix[j] =
1004             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
1005             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
1006         }
1007         if (s->avctx->intra_matrix)
1008             s->intra_matrix[j] = s->avctx->intra_matrix[i];
1009         if (s->avctx->inter_matrix)
1010             s->inter_matrix[j] = s->avctx->inter_matrix[i];
1011     }
1012
1013     /* precompute matrix */
1014     /* for mjpeg, we do include qscale in the matrix */
1015     if (s->out_format != FMT_MJPEG) {
1016         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
1017                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
1018                           31, 1);
1019         ff_convert_matrix(s, s->q_inter_matrix, s->q_inter_matrix16,
1020                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
1021                           31, 0);
1022     }
1023
1024     if (ff_rate_control_init(s) < 0)
1025         return -1;
1026
1027 #if FF_API_PRIVATE_OPT
1028     FF_DISABLE_DEPRECATION_WARNINGS
1029     if (avctx->brd_scale)
1030         s->brd_scale = avctx->brd_scale;
1031
1032     if (avctx->prediction_method)
1033         s->pred = avctx->prediction_method + 1;
1034     FF_ENABLE_DEPRECATION_WARNINGS
1035 #endif
1036
1037     if (s->b_frame_strategy == 2) {
1038         for (i = 0; i < s->max_b_frames + 2; i++) {
1039             s->tmp_frames[i] = av_frame_alloc();
1040             if (!s->tmp_frames[i])
1041                 return AVERROR(ENOMEM);
1042
1043             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
1044             s->tmp_frames[i]->width  = s->width  >> s->brd_scale;
1045             s->tmp_frames[i]->height = s->height >> s->brd_scale;
1046
1047             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
1048             if (ret < 0)
1049                 return ret;
1050         }
1051     }
1052
1053     cpb_props = ff_add_cpb_side_data(avctx);
1054     if (!cpb_props)
1055         return AVERROR(ENOMEM);
1056     cpb_props->max_bitrate = avctx->rc_max_rate;
1057     cpb_props->min_bitrate = avctx->rc_min_rate;
1058     cpb_props->avg_bitrate = avctx->bit_rate;
1059     cpb_props->buffer_size = avctx->rc_buffer_size;
1060
1061     return 0;
1062 fail:
1063     ff_mpv_encode_end(avctx);
1064     return AVERROR_UNKNOWN;
1065 }
1066
1067 av_cold int ff_mpv_encode_end(AVCodecContext *avctx)
1068 {
1069     MpegEncContext *s = avctx->priv_data;
1070     int i;
1071
1072     ff_rate_control_uninit(s);
1073
1074     ff_mpv_common_end(s);
1075     if (CONFIG_MJPEG_ENCODER &&
1076         s->out_format == FMT_MJPEG)
1077         ff_mjpeg_encode_close(s);
1078
1079     av_freep(&avctx->extradata);
1080
1081     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
1082         av_frame_free(&s->tmp_frames[i]);
1083
1084     ff_free_picture_tables(&s->new_picture);
1085     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1086
1087     av_freep(&s->avctx->stats_out);
1088     av_freep(&s->ac_stats);
1089
1090     if(s->q_chroma_intra_matrix   != s->q_intra_matrix  ) av_freep(&s->q_chroma_intra_matrix);
1091     if(s->q_chroma_intra_matrix16 != s->q_intra_matrix16) av_freep(&s->q_chroma_intra_matrix16);
1092     s->q_chroma_intra_matrix=   NULL;
1093     s->q_chroma_intra_matrix16= NULL;
1094     av_freep(&s->q_intra_matrix);
1095     av_freep(&s->q_inter_matrix);
1096     av_freep(&s->q_intra_matrix16);
1097     av_freep(&s->q_inter_matrix16);
1098     av_freep(&s->input_picture);
1099     av_freep(&s->reordered_input_picture);
1100     av_freep(&s->dct_offset);
1101
1102     return 0;
1103 }
1104
1105 static int get_sae(uint8_t *src, int ref, int stride)
1106 {
1107     int x,y;
1108     int acc = 0;
1109
1110     for (y = 0; y < 16; y++) {
1111         for (x = 0; x < 16; x++) {
1112             acc += FFABS(src[x + y * stride] - ref);
1113         }
1114     }
1115
1116     return acc;
1117 }
1118
1119 static int get_intra_count(MpegEncContext *s, uint8_t *src,
1120                            uint8_t *ref, int stride)
1121 {
1122     int x, y, w, h;
1123     int acc = 0;
1124
1125     w = s->width  & ~15;
1126     h = s->height & ~15;
1127
1128     for (y = 0; y < h; y += 16) {
1129         for (x = 0; x < w; x += 16) {
1130             int offset = x + y * stride;
1131             int sad  = s->mecc.sad[0](NULL, src + offset, ref + offset,
1132                                       stride, 16);
1133             int mean = (s->mpvencdsp.pix_sum(src + offset, stride) + 128) >> 8;
1134             int sae  = get_sae(src + offset, mean, stride);
1135
1136             acc += sae + 500 < sad;
1137         }
1138     }
1139     return acc;
1140 }
1141
1142 static int alloc_picture(MpegEncContext *s, Picture *pic, int shared)
1143 {
1144     return ff_alloc_picture(s->avctx, pic, &s->me, &s->sc, shared, 1,
1145                             s->chroma_x_shift, s->chroma_y_shift, s->out_format,
1146                             s->mb_stride, s->mb_width, s->mb_height, s->b8_stride,
1147                             &s->linesize, &s->uvlinesize);
1148 }
1149
1150 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
1151 {
1152     Picture *pic = NULL;
1153     int64_t pts;
1154     int i, display_picture_number = 0, ret;
1155     int encoding_delay = s->max_b_frames ? s->max_b_frames
1156                                          : (s->low_delay ? 0 : 1);
1157     int flush_offset = 1;
1158     int direct = 1;
1159
1160     if (pic_arg) {
1161         pts = pic_arg->pts;
1162         display_picture_number = s->input_picture_number++;
1163
1164         if (pts != AV_NOPTS_VALUE) {
1165             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1166                 int64_t last = s->user_specified_pts;
1167
1168                 if (pts <= last) {
1169                     av_log(s->avctx, AV_LOG_ERROR,
1170                            "Invalid pts (%"PRId64") <= last (%"PRId64")\n",
1171                            pts, last);
1172                     return AVERROR(EINVAL);
1173                 }
1174
1175                 if (!s->low_delay && display_picture_number == 1)
1176                     s->dts_delta = pts - last;
1177             }
1178             s->user_specified_pts = pts;
1179         } else {
1180             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1181                 s->user_specified_pts =
1182                 pts = s->user_specified_pts + 1;
1183                 av_log(s->avctx, AV_LOG_INFO,
1184                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
1185                        pts);
1186             } else {
1187                 pts = display_picture_number;
1188             }
1189         }
1190
1191         if (!pic_arg->buf[0] ||
1192             pic_arg->linesize[0] != s->linesize ||
1193             pic_arg->linesize[1] != s->uvlinesize ||
1194             pic_arg->linesize[2] != s->uvlinesize)
1195             direct = 0;
1196         if ((s->width & 15) || (s->height & 15))
1197             direct = 0;
1198         if (((intptr_t)(pic_arg->data[0])) & (STRIDE_ALIGN-1))
1199             direct = 0;
1200         if (s->linesize & (STRIDE_ALIGN-1))
1201             direct = 0;
1202
1203         ff_dlog(s->avctx, "%d %d %"PTRDIFF_SPECIFIER" %"PTRDIFF_SPECIFIER"\n", pic_arg->linesize[0],
1204                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
1205
1206         i = ff_find_unused_picture(s->avctx, s->picture, direct);
1207         if (i < 0)
1208             return i;
1209
1210         pic = &s->picture[i];
1211         pic->reference = 3;
1212
1213         if (direct) {
1214             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
1215                 return ret;
1216         }
1217         ret = alloc_picture(s, pic, direct);
1218         if (ret < 0)
1219             return ret;
1220
1221         if (!direct) {
1222             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1223                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1224                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1225                 // empty
1226             } else {
1227                 int h_chroma_shift, v_chroma_shift;
1228                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
1229                                                  &h_chroma_shift,
1230                                                  &v_chroma_shift);
1231
1232                 for (i = 0; i < 3; i++) {
1233                     int src_stride = pic_arg->linesize[i];
1234                     int dst_stride = i ? s->uvlinesize : s->linesize;
1235                     int h_shift = i ? h_chroma_shift : 0;
1236                     int v_shift = i ? v_chroma_shift : 0;
1237                     int w = s->width  >> h_shift;
1238                     int h = s->height >> v_shift;
1239                     uint8_t *src = pic_arg->data[i];
1240                     uint8_t *dst = pic->f->data[i];
1241                     int vpad = 16;
1242
1243                     if (   s->codec_id == AV_CODEC_ID_MPEG2VIDEO
1244                         && !s->progressive_sequence
1245                         && FFALIGN(s->height, 32) - s->height > 16)
1246                         vpad = 32;
1247
1248                     if (!s->avctx->rc_buffer_size)
1249                         dst += INPLACE_OFFSET;
1250
1251                     if (src_stride == dst_stride)
1252                         memcpy(dst, src, src_stride * h);
1253                     else {
1254                         int h2 = h;
1255                         uint8_t *dst2 = dst;
1256                         while (h2--) {
1257                             memcpy(dst2, src, w);
1258                             dst2 += dst_stride;
1259                             src += src_stride;
1260                         }
1261                     }
1262                     if ((s->width & 15) || (s->height & (vpad-1))) {
1263                         s->mpvencdsp.draw_edges(dst, dst_stride,
1264                                                 w, h,
1265                                                 16 >> h_shift,
1266                                                 vpad >> v_shift,
1267                                                 EDGE_BOTTOM);
1268                     }
1269                 }
1270                 emms_c();
1271             }
1272         }
1273         ret = av_frame_copy_props(pic->f, pic_arg);
1274         if (ret < 0)
1275             return ret;
1276
1277         pic->f->display_picture_number = display_picture_number;
1278         pic->f->pts = pts; // we set this here to avoid modifying pic_arg
1279     } else {
1280         /* Flushing: When we have not received enough input frames,
1281          * ensure s->input_picture[0] contains the first picture */
1282         for (flush_offset = 0; flush_offset < encoding_delay + 1; flush_offset++)
1283             if (s->input_picture[flush_offset])
1284                 break;
1285
1286         if (flush_offset <= 1)
1287             flush_offset = 1;
1288         else
1289             encoding_delay = encoding_delay - flush_offset + 1;
1290     }
1291
1292     /* shift buffer entries */
1293     for (i = flush_offset; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1294         s->input_picture[i - flush_offset] = s->input_picture[i];
1295
1296     s->input_picture[encoding_delay] = (Picture*) pic;
1297
1298     return 0;
1299 }
1300
1301 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1302 {
1303     int x, y, plane;
1304     int score = 0;
1305     int64_t score64 = 0;
1306
1307     for (plane = 0; plane < 3; plane++) {
1308         const int stride = p->f->linesize[plane];
1309         const int bw = plane ? 1 : 2;
1310         for (y = 0; y < s->mb_height * bw; y++) {
1311             for (x = 0; x < s->mb_width * bw; x++) {
1312                 int off = p->shared ? 0 : 16;
1313                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1314                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1315                 int v = s->mecc.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1316
1317                 switch (FFABS(s->frame_skip_exp)) {
1318                 case 0: score    =  FFMAX(score, v);          break;
1319                 case 1: score   += FFABS(v);                  break;
1320                 case 2: score64 += v * (int64_t)v;                       break;
1321                 case 3: score64 += FFABS(v * (int64_t)v * v);            break;
1322                 case 4: score64 += (v * (int64_t)v) * (v * (int64_t)v);  break;
1323                 }
1324             }
1325         }
1326     }
1327     emms_c();
1328
1329     if (score)
1330         score64 = score;
1331     if (s->frame_skip_exp < 0)
1332         score64 = pow(score64 / (double)(s->mb_width * s->mb_height),
1333                       -1.0/s->frame_skip_exp);
1334
1335     if (score64 < s->frame_skip_threshold)
1336         return 1;
1337     if (score64 < ((s->frame_skip_factor * (int64_t) s->lambda) >> 8))
1338         return 1;
1339     return 0;
1340 }
1341
1342 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1343 {
1344     AVPacket pkt = { 0 };
1345     int ret;
1346     int size = 0;
1347
1348     av_init_packet(&pkt);
1349
1350     ret = avcodec_send_frame(c, frame);
1351     if (ret < 0)
1352         return ret;
1353
1354     do {
1355         ret = avcodec_receive_packet(c, &pkt);
1356         if (ret >= 0) {
1357             size += pkt.size;
1358             av_packet_unref(&pkt);
1359         } else if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
1360             return ret;
1361     } while (ret >= 0);
1362
1363     return size;
1364 }
1365
1366 static int estimate_best_b_count(MpegEncContext *s)
1367 {
1368     const AVCodec *codec = avcodec_find_encoder(s->avctx->codec_id);
1369     const int scale = s->brd_scale;
1370     int width  = s->width  >> scale;
1371     int height = s->height >> scale;
1372     int i, j, out_size, p_lambda, b_lambda, lambda2;
1373     int64_t best_rd  = INT64_MAX;
1374     int best_b_count = -1;
1375     int ret = 0;
1376
1377     av_assert0(scale >= 0 && scale <= 3);
1378
1379     //emms_c();
1380     //s->next_picture_ptr->quality;
1381     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1382     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1383     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1384     if (!b_lambda) // FIXME we should do this somewhere else
1385         b_lambda = p_lambda;
1386     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1387                FF_LAMBDA_SHIFT;
1388
1389     for (i = 0; i < s->max_b_frames + 2; i++) {
1390         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1391                                                 s->next_picture_ptr;
1392         uint8_t *data[4];
1393
1394         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1395             pre_input = *pre_input_ptr;
1396             memcpy(data, pre_input_ptr->f->data, sizeof(data));
1397
1398             if (!pre_input.shared && i) {
1399                 data[0] += INPLACE_OFFSET;
1400                 data[1] += INPLACE_OFFSET;
1401                 data[2] += INPLACE_OFFSET;
1402             }
1403
1404             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[0],
1405                                        s->tmp_frames[i]->linesize[0],
1406                                        data[0],
1407                                        pre_input.f->linesize[0],
1408                                        width, height);
1409             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[1],
1410                                        s->tmp_frames[i]->linesize[1],
1411                                        data[1],
1412                                        pre_input.f->linesize[1],
1413                                        width >> 1, height >> 1);
1414             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[2],
1415                                        s->tmp_frames[i]->linesize[2],
1416                                        data[2],
1417                                        pre_input.f->linesize[2],
1418                                        width >> 1, height >> 1);
1419         }
1420     }
1421
1422     for (j = 0; j < s->max_b_frames + 1; j++) {
1423         AVCodecContext *c;
1424         int64_t rd = 0;
1425
1426         if (!s->input_picture[j])
1427             break;
1428
1429         c = avcodec_alloc_context3(NULL);
1430         if (!c)
1431             return AVERROR(ENOMEM);
1432
1433         c->width        = width;
1434         c->height       = height;
1435         c->flags        = AV_CODEC_FLAG_QSCALE | AV_CODEC_FLAG_PSNR;
1436         c->flags       |= s->avctx->flags & AV_CODEC_FLAG_QPEL;
1437         c->mb_decision  = s->avctx->mb_decision;
1438         c->me_cmp       = s->avctx->me_cmp;
1439         c->mb_cmp       = s->avctx->mb_cmp;
1440         c->me_sub_cmp   = s->avctx->me_sub_cmp;
1441         c->pix_fmt      = AV_PIX_FMT_YUV420P;
1442         c->time_base    = s->avctx->time_base;
1443         c->max_b_frames = s->max_b_frames;
1444
1445         ret = avcodec_open2(c, codec, NULL);
1446         if (ret < 0)
1447             goto fail;
1448
1449         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1450         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1451
1452         out_size = encode_frame(c, s->tmp_frames[0]);
1453         if (out_size < 0) {
1454             ret = out_size;
1455             goto fail;
1456         }
1457
1458         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1459
1460         for (i = 0; i < s->max_b_frames + 1; i++) {
1461             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1462
1463             s->tmp_frames[i + 1]->pict_type = is_p ?
1464                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1465             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1466
1467             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1468             if (out_size < 0) {
1469                 ret = out_size;
1470                 goto fail;
1471             }
1472
1473             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1474         }
1475
1476         /* get the delayed frames */
1477         out_size = encode_frame(c, NULL);
1478         if (out_size < 0) {
1479             ret = out_size;
1480             goto fail;
1481         }
1482         rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1483
1484         rd += c->error[0] + c->error[1] + c->error[2];
1485
1486         if (rd < best_rd) {
1487             best_rd = rd;
1488             best_b_count = j;
1489         }
1490
1491 fail:
1492         avcodec_free_context(&c);
1493         if (ret < 0)
1494             return ret;
1495     }
1496
1497     return best_b_count;
1498 }
1499
1500 static int select_input_picture(MpegEncContext *s)
1501 {
1502     int i, ret;
1503
1504     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1505         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1506     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1507
1508     /* set next picture type & ordering */
1509     if (!s->reordered_input_picture[0] && s->input_picture[0]) {
1510         if (s->frame_skip_threshold || s->frame_skip_factor) {
1511             if (s->picture_in_gop_number < s->gop_size &&
1512                 s->next_picture_ptr &&
1513                 skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1514                 // FIXME check that the gop check above is +-1 correct
1515                 av_frame_unref(s->input_picture[0]->f);
1516
1517                 ff_vbv_update(s, 0);
1518
1519                 goto no_output_pic;
1520             }
1521         }
1522
1523         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1524             !s->next_picture_ptr || s->intra_only) {
1525             s->reordered_input_picture[0] = s->input_picture[0];
1526             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1527             s->reordered_input_picture[0]->f->coded_picture_number =
1528                 s->coded_picture_number++;
1529         } else {
1530             int b_frames = 0;
1531
1532             if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
1533                 for (i = 0; i < s->max_b_frames + 1; i++) {
1534                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1535
1536                     if (pict_num >= s->rc_context.num_entries)
1537                         break;
1538                     if (!s->input_picture[i]) {
1539                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1540                         break;
1541                     }
1542
1543                     s->input_picture[i]->f->pict_type =
1544                         s->rc_context.entry[pict_num].new_pict_type;
1545                 }
1546             }
1547
1548             if (s->b_frame_strategy == 0) {
1549                 b_frames = s->max_b_frames;
1550                 while (b_frames && !s->input_picture[b_frames])
1551                     b_frames--;
1552             } else if (s->b_frame_strategy == 1) {
1553                 for (i = 1; i < s->max_b_frames + 1; i++) {
1554                     if (s->input_picture[i] &&
1555                         s->input_picture[i]->b_frame_score == 0) {
1556                         s->input_picture[i]->b_frame_score =
1557                             get_intra_count(s,
1558                                             s->input_picture[i    ]->f->data[0],
1559                                             s->input_picture[i - 1]->f->data[0],
1560                                             s->linesize) + 1;
1561                     }
1562                 }
1563                 for (i = 0; i < s->max_b_frames + 1; i++) {
1564                     if (!s->input_picture[i] ||
1565                         s->input_picture[i]->b_frame_score - 1 >
1566                             s->mb_num / s->b_sensitivity)
1567                         break;
1568                 }
1569
1570                 b_frames = FFMAX(0, i - 1);
1571
1572                 /* reset scores */
1573                 for (i = 0; i < b_frames + 1; i++) {
1574                     s->input_picture[i]->b_frame_score = 0;
1575                 }
1576             } else if (s->b_frame_strategy == 2) {
1577                 b_frames = estimate_best_b_count(s);
1578                 if (b_frames < 0)
1579                     return b_frames;
1580             }
1581
1582             emms_c();
1583
1584             for (i = b_frames - 1; i >= 0; i--) {
1585                 int type = s->input_picture[i]->f->pict_type;
1586                 if (type && type != AV_PICTURE_TYPE_B)
1587                     b_frames = i;
1588             }
1589             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1590                 b_frames == s->max_b_frames) {
1591                 av_log(s->avctx, AV_LOG_ERROR,
1592                        "warning, too many B-frames in a row\n");
1593             }
1594
1595             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1596                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1597                     s->gop_size > s->picture_in_gop_number) {
1598                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1599                 } else {
1600                     if (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)
1601                         b_frames = 0;
1602                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1603                 }
1604             }
1605
1606             if ((s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP) && b_frames &&
1607                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1608                 b_frames--;
1609
1610             s->reordered_input_picture[0] = s->input_picture[b_frames];
1611             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1612                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1613             s->reordered_input_picture[0]->f->coded_picture_number =
1614                 s->coded_picture_number++;
1615             for (i = 0; i < b_frames; i++) {
1616                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1617                 s->reordered_input_picture[i + 1]->f->pict_type =
1618                     AV_PICTURE_TYPE_B;
1619                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1620                     s->coded_picture_number++;
1621             }
1622         }
1623     }
1624 no_output_pic:
1625     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1626
1627     if (s->reordered_input_picture[0]) {
1628         s->reordered_input_picture[0]->reference =
1629            s->reordered_input_picture[0]->f->pict_type !=
1630                AV_PICTURE_TYPE_B ? 3 : 0;
1631
1632         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->new_picture, s->reordered_input_picture[0])))
1633             return ret;
1634
1635         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1636             // input is a shared pix, so we can't modify it -> allocate a new
1637             // one & ensure that the shared one is reuseable
1638
1639             Picture *pic;
1640             int i = ff_find_unused_picture(s->avctx, s->picture, 0);
1641             if (i < 0)
1642                 return i;
1643             pic = &s->picture[i];
1644
1645             pic->reference = s->reordered_input_picture[0]->reference;
1646             if (alloc_picture(s, pic, 0) < 0) {
1647                 return -1;
1648             }
1649
1650             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1651             if (ret < 0)
1652                 return ret;
1653
1654             /* mark us unused / free shared pic */
1655             av_frame_unref(s->reordered_input_picture[0]->f);
1656             s->reordered_input_picture[0]->shared = 0;
1657
1658             s->current_picture_ptr = pic;
1659         } else {
1660             // input is not a shared pix -> reuse buffer for current_pix
1661             s->current_picture_ptr = s->reordered_input_picture[0];
1662             for (i = 0; i < 4; i++) {
1663                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1664             }
1665         }
1666         ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1667         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1668                                        s->current_picture_ptr)) < 0)
1669             return ret;
1670
1671         s->picture_number = s->new_picture.f->display_picture_number;
1672     }
1673     return 0;
1674 }
1675
1676 static void frame_end(MpegEncContext *s)
1677 {
1678     if (s->unrestricted_mv &&
1679         s->current_picture.reference &&
1680         !s->intra_only) {
1681         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1682         int hshift = desc->log2_chroma_w;
1683         int vshift = desc->log2_chroma_h;
1684         s->mpvencdsp.draw_edges(s->current_picture.f->data[0],
1685                                 s->current_picture.f->linesize[0],
1686                                 s->h_edge_pos, s->v_edge_pos,
1687                                 EDGE_WIDTH, EDGE_WIDTH,
1688                                 EDGE_TOP | EDGE_BOTTOM);
1689         s->mpvencdsp.draw_edges(s->current_picture.f->data[1],
1690                                 s->current_picture.f->linesize[1],
1691                                 s->h_edge_pos >> hshift,
1692                                 s->v_edge_pos >> vshift,
1693                                 EDGE_WIDTH >> hshift,
1694                                 EDGE_WIDTH >> vshift,
1695                                 EDGE_TOP | EDGE_BOTTOM);
1696         s->mpvencdsp.draw_edges(s->current_picture.f->data[2],
1697                                 s->current_picture.f->linesize[2],
1698                                 s->h_edge_pos >> hshift,
1699                                 s->v_edge_pos >> vshift,
1700                                 EDGE_WIDTH >> hshift,
1701                                 EDGE_WIDTH >> vshift,
1702                                 EDGE_TOP | EDGE_BOTTOM);
1703     }
1704
1705     emms_c();
1706
1707     s->last_pict_type                 = s->pict_type;
1708     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1709     if (s->pict_type!= AV_PICTURE_TYPE_B)
1710         s->last_non_b_pict_type = s->pict_type;
1711
1712 #if FF_API_CODED_FRAME
1713 FF_DISABLE_DEPRECATION_WARNINGS
1714     av_frame_unref(s->avctx->coded_frame);
1715     av_frame_copy_props(s->avctx->coded_frame, s->current_picture.f);
1716 FF_ENABLE_DEPRECATION_WARNINGS
1717 #endif
1718 #if FF_API_ERROR_FRAME
1719 FF_DISABLE_DEPRECATION_WARNINGS
1720     memcpy(s->current_picture.f->error, s->current_picture.encoding_error,
1721            sizeof(s->current_picture.encoding_error));
1722 FF_ENABLE_DEPRECATION_WARNINGS
1723 #endif
1724 }
1725
1726 static void update_noise_reduction(MpegEncContext *s)
1727 {
1728     int intra, i;
1729
1730     for (intra = 0; intra < 2; intra++) {
1731         if (s->dct_count[intra] > (1 << 16)) {
1732             for (i = 0; i < 64; i++) {
1733                 s->dct_error_sum[intra][i] >>= 1;
1734             }
1735             s->dct_count[intra] >>= 1;
1736         }
1737
1738         for (i = 0; i < 64; i++) {
1739             s->dct_offset[intra][i] = (s->noise_reduction *
1740                                        s->dct_count[intra] +
1741                                        s->dct_error_sum[intra][i] / 2) /
1742                                       (s->dct_error_sum[intra][i] + 1);
1743         }
1744     }
1745 }
1746
1747 static int frame_start(MpegEncContext *s)
1748 {
1749     int ret;
1750
1751     /* mark & release old frames */
1752     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1753         s->last_picture_ptr != s->next_picture_ptr &&
1754         s->last_picture_ptr->f->buf[0]) {
1755         ff_mpeg_unref_picture(s->avctx, s->last_picture_ptr);
1756     }
1757
1758     s->current_picture_ptr->f->pict_type = s->pict_type;
1759     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1760
1761     ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1762     if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1763                                    s->current_picture_ptr)) < 0)
1764         return ret;
1765
1766     if (s->pict_type != AV_PICTURE_TYPE_B) {
1767         s->last_picture_ptr = s->next_picture_ptr;
1768         if (!s->droppable)
1769             s->next_picture_ptr = s->current_picture_ptr;
1770     }
1771
1772     if (s->last_picture_ptr) {
1773         ff_mpeg_unref_picture(s->avctx, &s->last_picture);
1774         if (s->last_picture_ptr->f->buf[0] &&
1775             (ret = ff_mpeg_ref_picture(s->avctx, &s->last_picture,
1776                                        s->last_picture_ptr)) < 0)
1777             return ret;
1778     }
1779     if (s->next_picture_ptr) {
1780         ff_mpeg_unref_picture(s->avctx, &s->next_picture);
1781         if (s->next_picture_ptr->f->buf[0] &&
1782             (ret = ff_mpeg_ref_picture(s->avctx, &s->next_picture,
1783                                        s->next_picture_ptr)) < 0)
1784             return ret;
1785     }
1786
1787     if (s->picture_structure!= PICT_FRAME) {
1788         int i;
1789         for (i = 0; i < 4; i++) {
1790             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1791                 s->current_picture.f->data[i] +=
1792                     s->current_picture.f->linesize[i];
1793             }
1794             s->current_picture.f->linesize[i] *= 2;
1795             s->last_picture.f->linesize[i]    *= 2;
1796             s->next_picture.f->linesize[i]    *= 2;
1797         }
1798     }
1799
1800     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1801         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1802         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1803     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1804         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1805         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1806     } else {
1807         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1808         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1809     }
1810
1811     if (s->dct_error_sum) {
1812         av_assert2(s->noise_reduction && s->encoding);
1813         update_noise_reduction(s);
1814     }
1815
1816     return 0;
1817 }
1818
1819 int ff_mpv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1820                           const AVFrame *pic_arg, int *got_packet)
1821 {
1822     MpegEncContext *s = avctx->priv_data;
1823     int i, stuffing_count, ret;
1824     int context_count = s->slice_context_count;
1825
1826     s->vbv_ignore_qmax = 0;
1827
1828     s->picture_in_gop_number++;
1829
1830     if (load_input_picture(s, pic_arg) < 0)
1831         return -1;
1832
1833     if (select_input_picture(s) < 0) {
1834         return -1;
1835     }
1836
1837     /* output? */
1838     if (s->new_picture.f->data[0]) {
1839         int growing_buffer = context_count == 1 && !pkt->data && !s->data_partitioning;
1840         int pkt_size = growing_buffer ? FFMAX(s->mb_width*s->mb_height*64+10000, avctx->internal->byte_buffer_size) - AV_INPUT_BUFFER_PADDING_SIZE
1841                                               :
1842                                               s->mb_width*s->mb_height*(MAX_MB_BYTES+100)+10000;
1843         if ((ret = ff_alloc_packet2(avctx, pkt, pkt_size, 0)) < 0)
1844             return ret;
1845         if (s->mb_info) {
1846             s->mb_info_ptr = av_packet_new_side_data(pkt,
1847                                  AV_PKT_DATA_H263_MB_INFO,
1848                                  s->mb_width*s->mb_height*12);
1849             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1850         }
1851
1852         for (i = 0; i < context_count; i++) {
1853             int start_y = s->thread_context[i]->start_mb_y;
1854             int   end_y = s->thread_context[i]->  end_mb_y;
1855             int h       = s->mb_height;
1856             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1857             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1858
1859             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1860         }
1861
1862         s->pict_type = s->new_picture.f->pict_type;
1863         //emms_c();
1864         ret = frame_start(s);
1865         if (ret < 0)
1866             return ret;
1867 vbv_retry:
1868         ret = encode_picture(s, s->picture_number);
1869         if (growing_buffer) {
1870             av_assert0(s->pb.buf == avctx->internal->byte_buffer);
1871             pkt->data = s->pb.buf;
1872             pkt->size = avctx->internal->byte_buffer_size;
1873         }
1874         if (ret < 0)
1875             return -1;
1876
1877 #if FF_API_STAT_BITS
1878 FF_DISABLE_DEPRECATION_WARNINGS
1879         avctx->header_bits = s->header_bits;
1880         avctx->mv_bits     = s->mv_bits;
1881         avctx->misc_bits   = s->misc_bits;
1882         avctx->i_tex_bits  = s->i_tex_bits;
1883         avctx->p_tex_bits  = s->p_tex_bits;
1884         avctx->i_count     = s->i_count;
1885         // FIXME f/b_count in avctx
1886         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1887         avctx->skip_count  = s->skip_count;
1888 FF_ENABLE_DEPRECATION_WARNINGS
1889 #endif
1890
1891         frame_end(s);
1892
1893         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1894             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1895
1896         if (avctx->rc_buffer_size) {
1897             RateControlContext *rcc = &s->rc_context;
1898             int max_size = FFMAX(rcc->buffer_index * avctx->rc_max_available_vbv_use, rcc->buffer_index - 500);
1899             int hq = (s->avctx->mb_decision == FF_MB_DECISION_RD || s->avctx->trellis);
1900             int min_step = hq ? 1 : (1<<(FF_LAMBDA_SHIFT + 7))/139;
1901
1902             if (put_bits_count(&s->pb) > max_size &&
1903                 s->lambda < s->lmax) {
1904                 s->next_lambda = FFMAX(s->lambda + min_step, s->lambda *
1905                                        (s->qscale + 1) / s->qscale);
1906                 if (s->adaptive_quant) {
1907                     int i;
1908                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1909                         s->lambda_table[i] =
1910                             FFMAX(s->lambda_table[i] + min_step,
1911                                   s->lambda_table[i] * (s->qscale + 1) /
1912                                   s->qscale);
1913                 }
1914                 s->mb_skipped = 0;        // done in frame_start()
1915                 // done in encode_picture() so we must undo it
1916                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1917                     if (s->flipflop_rounding          ||
1918                         s->codec_id == AV_CODEC_ID_H263P ||
1919                         s->codec_id == AV_CODEC_ID_MPEG4)
1920                         s->no_rounding ^= 1;
1921                 }
1922                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1923                     s->time_base       = s->last_time_base;
1924                     s->last_non_b_time = s->time - s->pp_time;
1925                 }
1926                 for (i = 0; i < context_count; i++) {
1927                     PutBitContext *pb = &s->thread_context[i]->pb;
1928                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1929                 }
1930                 s->vbv_ignore_qmax = 1;
1931                 av_log(s->avctx, AV_LOG_VERBOSE, "reencoding frame due to VBV\n");
1932                 goto vbv_retry;
1933             }
1934
1935             av_assert0(s->avctx->rc_max_rate);
1936         }
1937
1938         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
1939             ff_write_pass1_stats(s);
1940
1941         for (i = 0; i < 4; i++) {
1942             s->current_picture_ptr->encoding_error[i] = s->current_picture.encoding_error[i];
1943             avctx->error[i] += s->current_picture_ptr->encoding_error[i];
1944         }
1945         ff_side_data_set_encoder_stats(pkt, s->current_picture.f->quality,
1946                                        s->current_picture_ptr->encoding_error,
1947                                        (s->avctx->flags&AV_CODEC_FLAG_PSNR) ? 4 : 0,
1948                                        s->pict_type);
1949
1950         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
1951             assert(put_bits_count(&s->pb) == s->header_bits + s->mv_bits +
1952                                              s->misc_bits + s->i_tex_bits +
1953                                              s->p_tex_bits);
1954         flush_put_bits(&s->pb);
1955         s->frame_bits  = put_bits_count(&s->pb);
1956
1957         stuffing_count = ff_vbv_update(s, s->frame_bits);
1958         s->stuffing_bits = 8*stuffing_count;
1959         if (stuffing_count) {
1960             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1961                     stuffing_count + 50) {
1962                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1963                 return -1;
1964             }
1965
1966             switch (s->codec_id) {
1967             case AV_CODEC_ID_MPEG1VIDEO:
1968             case AV_CODEC_ID_MPEG2VIDEO:
1969                 while (stuffing_count--) {
1970                     put_bits(&s->pb, 8, 0);
1971                 }
1972             break;
1973             case AV_CODEC_ID_MPEG4:
1974                 put_bits(&s->pb, 16, 0);
1975                 put_bits(&s->pb, 16, 0x1C3);
1976                 stuffing_count -= 4;
1977                 while (stuffing_count--) {
1978                     put_bits(&s->pb, 8, 0xFF);
1979                 }
1980             break;
1981             default:
1982                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1983             }
1984             flush_put_bits(&s->pb);
1985             s->frame_bits  = put_bits_count(&s->pb);
1986         }
1987
1988         /* update MPEG-1/2 vbv_delay for CBR */
1989         if (s->avctx->rc_max_rate                          &&
1990             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1991             s->out_format == FMT_MPEG1                     &&
1992             90000LL * (avctx->rc_buffer_size - 1) <=
1993                 s->avctx->rc_max_rate * 0xFFFFLL) {
1994             AVCPBProperties *props;
1995             size_t props_size;
1996
1997             int vbv_delay, min_delay;
1998             double inbits  = s->avctx->rc_max_rate *
1999                              av_q2d(s->avctx->time_base);
2000             int    minbits = s->frame_bits - 8 *
2001                              (s->vbv_delay_ptr - s->pb.buf - 1);
2002             double bits    = s->rc_context.buffer_index + minbits - inbits;
2003
2004             if (bits < 0)
2005                 av_log(s->avctx, AV_LOG_ERROR,
2006                        "Internal error, negative bits\n");
2007
2008             assert(s->repeat_first_field == 0);
2009
2010             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
2011             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
2012                         s->avctx->rc_max_rate;
2013
2014             vbv_delay = FFMAX(vbv_delay, min_delay);
2015
2016             av_assert0(vbv_delay < 0xFFFF);
2017
2018             s->vbv_delay_ptr[0] &= 0xF8;
2019             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
2020             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
2021             s->vbv_delay_ptr[2] &= 0x07;
2022             s->vbv_delay_ptr[2] |= vbv_delay << 3;
2023
2024             props = av_cpb_properties_alloc(&props_size);
2025             if (!props)
2026                 return AVERROR(ENOMEM);
2027             props->vbv_delay = vbv_delay * 300;
2028
2029             ret = av_packet_add_side_data(pkt, AV_PKT_DATA_CPB_PROPERTIES,
2030                                           (uint8_t*)props, props_size);
2031             if (ret < 0) {
2032                 av_freep(&props);
2033                 return ret;
2034             }
2035
2036 #if FF_API_VBV_DELAY
2037 FF_DISABLE_DEPRECATION_WARNINGS
2038             avctx->vbv_delay     = vbv_delay * 300;
2039 FF_ENABLE_DEPRECATION_WARNINGS
2040 #endif
2041         }
2042         s->total_bits     += s->frame_bits;
2043 #if FF_API_STAT_BITS
2044 FF_DISABLE_DEPRECATION_WARNINGS
2045         avctx->frame_bits  = s->frame_bits;
2046 FF_ENABLE_DEPRECATION_WARNINGS
2047 #endif
2048
2049
2050         pkt->pts = s->current_picture.f->pts;
2051         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
2052             if (!s->current_picture.f->coded_picture_number)
2053                 pkt->dts = pkt->pts - s->dts_delta;
2054             else
2055                 pkt->dts = s->reordered_pts;
2056             s->reordered_pts = pkt->pts;
2057         } else
2058             pkt->dts = pkt->pts;
2059         if (s->current_picture.f->key_frame)
2060             pkt->flags |= AV_PKT_FLAG_KEY;
2061         if (s->mb_info)
2062             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
2063     } else {
2064         s->frame_bits = 0;
2065     }
2066
2067     /* release non-reference frames */
2068     for (i = 0; i < MAX_PICTURE_COUNT; i++) {
2069         if (!s->picture[i].reference)
2070             ff_mpeg_unref_picture(s->avctx, &s->picture[i]);
2071     }
2072
2073     av_assert1((s->frame_bits & 7) == 0);
2074
2075     pkt->size = s->frame_bits / 8;
2076     *got_packet = !!pkt->size;
2077     return 0;
2078 }
2079
2080 static inline void dct_single_coeff_elimination(MpegEncContext *s,
2081                                                 int n, int threshold)
2082 {
2083     static const char tab[64] = {
2084         3, 2, 2, 1, 1, 1, 1, 1,
2085         1, 1, 1, 1, 1, 1, 1, 1,
2086         1, 1, 1, 1, 1, 1, 1, 1,
2087         0, 0, 0, 0, 0, 0, 0, 0,
2088         0, 0, 0, 0, 0, 0, 0, 0,
2089         0, 0, 0, 0, 0, 0, 0, 0,
2090         0, 0, 0, 0, 0, 0, 0, 0,
2091         0, 0, 0, 0, 0, 0, 0, 0
2092     };
2093     int score = 0;
2094     int run = 0;
2095     int i;
2096     int16_t *block = s->block[n];
2097     const int last_index = s->block_last_index[n];
2098     int skip_dc;
2099
2100     if (threshold < 0) {
2101         skip_dc = 0;
2102         threshold = -threshold;
2103     } else
2104         skip_dc = 1;
2105
2106     /* Are all we could set to zero already zero? */
2107     if (last_index <= skip_dc - 1)
2108         return;
2109
2110     for (i = 0; i <= last_index; i++) {
2111         const int j = s->intra_scantable.permutated[i];
2112         const int level = FFABS(block[j]);
2113         if (level == 1) {
2114             if (skip_dc && i == 0)
2115                 continue;
2116             score += tab[run];
2117             run = 0;
2118         } else if (level > 1) {
2119             return;
2120         } else {
2121             run++;
2122         }
2123     }
2124     if (score >= threshold)
2125         return;
2126     for (i = skip_dc; i <= last_index; i++) {
2127         const int j = s->intra_scantable.permutated[i];
2128         block[j] = 0;
2129     }
2130     if (block[0])
2131         s->block_last_index[n] = 0;
2132     else
2133         s->block_last_index[n] = -1;
2134 }
2135
2136 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
2137                                int last_index)
2138 {
2139     int i;
2140     const int maxlevel = s->max_qcoeff;
2141     const int minlevel = s->min_qcoeff;
2142     int overflow = 0;
2143
2144     if (s->mb_intra) {
2145         i = 1; // skip clipping of intra dc
2146     } else
2147         i = 0;
2148
2149     for (; i <= last_index; i++) {
2150         const int j = s->intra_scantable.permutated[i];
2151         int level = block[j];
2152
2153         if (level > maxlevel) {
2154             level = maxlevel;
2155             overflow++;
2156         } else if (level < minlevel) {
2157             level = minlevel;
2158             overflow++;
2159         }
2160
2161         block[j] = level;
2162     }
2163
2164     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
2165         av_log(s->avctx, AV_LOG_INFO,
2166                "warning, clipping %d dct coefficients to %d..%d\n",
2167                overflow, minlevel, maxlevel);
2168 }
2169
2170 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
2171 {
2172     int x, y;
2173     // FIXME optimize
2174     for (y = 0; y < 8; y++) {
2175         for (x = 0; x < 8; x++) {
2176             int x2, y2;
2177             int sum = 0;
2178             int sqr = 0;
2179             int count = 0;
2180
2181             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
2182                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
2183                     int v = ptr[x2 + y2 * stride];
2184                     sum += v;
2185                     sqr += v * v;
2186                     count++;
2187                 }
2188             }
2189             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
2190         }
2191     }
2192 }
2193
2194 static av_always_inline void encode_mb_internal(MpegEncContext *s,
2195                                                 int motion_x, int motion_y,
2196                                                 int mb_block_height,
2197                                                 int mb_block_width,
2198                                                 int mb_block_count)
2199 {
2200     int16_t weight[12][64];
2201     int16_t orig[12][64];
2202     const int mb_x = s->mb_x;
2203     const int mb_y = s->mb_y;
2204     int i;
2205     int skip_dct[12];
2206     int dct_offset = s->linesize * 8; // default for progressive frames
2207     int uv_dct_offset = s->uvlinesize * 8;
2208     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
2209     ptrdiff_t wrap_y, wrap_c;
2210
2211     for (i = 0; i < mb_block_count; i++)
2212         skip_dct[i] = s->skipdct;
2213
2214     if (s->adaptive_quant) {
2215         const int last_qp = s->qscale;
2216         const int mb_xy = mb_x + mb_y * s->mb_stride;
2217
2218         s->lambda = s->lambda_table[mb_xy];
2219         update_qscale(s);
2220
2221         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
2222             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
2223             s->dquant = s->qscale - last_qp;
2224
2225             if (s->out_format == FMT_H263) {
2226                 s->dquant = av_clip(s->dquant, -2, 2);
2227
2228                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
2229                     if (!s->mb_intra) {
2230                         if (s->pict_type == AV_PICTURE_TYPE_B) {
2231                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
2232                                 s->dquant = 0;
2233                         }
2234                         if (s->mv_type == MV_TYPE_8X8)
2235                             s->dquant = 0;
2236                     }
2237                 }
2238             }
2239         }
2240         ff_set_qscale(s, last_qp + s->dquant);
2241     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
2242         ff_set_qscale(s, s->qscale + s->dquant);
2243
2244     wrap_y = s->linesize;
2245     wrap_c = s->uvlinesize;
2246     ptr_y  = s->new_picture.f->data[0] +
2247              (mb_y * 16 * wrap_y)              + mb_x * 16;
2248     ptr_cb = s->new_picture.f->data[1] +
2249              (mb_y * mb_block_height * wrap_c) + mb_x * mb_block_width;
2250     ptr_cr = s->new_picture.f->data[2] +
2251              (mb_y * mb_block_height * wrap_c) + mb_x * mb_block_width;
2252
2253     if((mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) && s->codec_id != AV_CODEC_ID_AMV){
2254         uint8_t *ebuf = s->sc.edge_emu_buffer + 38 * wrap_y;
2255         int cw = (s->width  + s->chroma_x_shift) >> s->chroma_x_shift;
2256         int ch = (s->height + s->chroma_y_shift) >> s->chroma_y_shift;
2257         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
2258                                  wrap_y, wrap_y,
2259                                  16, 16, mb_x * 16, mb_y * 16,
2260                                  s->width, s->height);
2261         ptr_y = ebuf;
2262         s->vdsp.emulated_edge_mc(ebuf + 16 * wrap_y, ptr_cb,
2263                                  wrap_c, wrap_c,
2264                                  mb_block_width, mb_block_height,
2265                                  mb_x * mb_block_width, mb_y * mb_block_height,
2266                                  cw, ch);
2267         ptr_cb = ebuf + 16 * wrap_y;
2268         s->vdsp.emulated_edge_mc(ebuf + 16 * wrap_y + 16, ptr_cr,
2269                                  wrap_c, wrap_c,
2270                                  mb_block_width, mb_block_height,
2271                                  mb_x * mb_block_width, mb_y * mb_block_height,
2272                                  cw, ch);
2273         ptr_cr = ebuf + 16 * wrap_y + 16;
2274     }
2275
2276     if (s->mb_intra) {
2277         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2278             int progressive_score, interlaced_score;
2279
2280             s->interlaced_dct = 0;
2281             progressive_score = s->mecc.ildct_cmp[4](s, ptr_y, NULL, wrap_y, 8) +
2282                                 s->mecc.ildct_cmp[4](s, ptr_y + wrap_y * 8,
2283                                                      NULL, wrap_y, 8) - 400;
2284
2285             if (progressive_score > 0) {
2286                 interlaced_score = s->mecc.ildct_cmp[4](s, ptr_y,
2287                                                         NULL, wrap_y * 2, 8) +
2288                                    s->mecc.ildct_cmp[4](s, ptr_y + wrap_y,
2289                                                         NULL, wrap_y * 2, 8);
2290                 if (progressive_score > interlaced_score) {
2291                     s->interlaced_dct = 1;
2292
2293                     dct_offset = wrap_y;
2294                     uv_dct_offset = wrap_c;
2295                     wrap_y <<= 1;
2296                     if (s->chroma_format == CHROMA_422 ||
2297                         s->chroma_format == CHROMA_444)
2298                         wrap_c <<= 1;
2299                 }
2300             }
2301         }
2302
2303         s->pdsp.get_pixels(s->block[0], ptr_y,                  wrap_y);
2304         s->pdsp.get_pixels(s->block[1], ptr_y + 8,              wrap_y);
2305         s->pdsp.get_pixels(s->block[2], ptr_y + dct_offset,     wrap_y);
2306         s->pdsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
2307
2308         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2309             skip_dct[4] = 1;
2310             skip_dct[5] = 1;
2311         } else {
2312             s->pdsp.get_pixels(s->block[4], ptr_cb, wrap_c);
2313             s->pdsp.get_pixels(s->block[5], ptr_cr, wrap_c);
2314             if (!s->chroma_y_shift && s->chroma_x_shift) { /* 422 */
2315                 s->pdsp.get_pixels(s->block[6], ptr_cb + uv_dct_offset, wrap_c);
2316                 s->pdsp.get_pixels(s->block[7], ptr_cr + uv_dct_offset, wrap_c);
2317             } else if (!s->chroma_y_shift && !s->chroma_x_shift) { /* 444 */
2318                 s->pdsp.get_pixels(s->block[ 6], ptr_cb + 8, wrap_c);
2319                 s->pdsp.get_pixels(s->block[ 7], ptr_cr + 8, wrap_c);
2320                 s->pdsp.get_pixels(s->block[ 8], ptr_cb + uv_dct_offset, wrap_c);
2321                 s->pdsp.get_pixels(s->block[ 9], ptr_cr + uv_dct_offset, wrap_c);
2322                 s->pdsp.get_pixels(s->block[10], ptr_cb + uv_dct_offset + 8, wrap_c);
2323                 s->pdsp.get_pixels(s->block[11], ptr_cr + uv_dct_offset + 8, wrap_c);
2324             }
2325         }
2326     } else {
2327         op_pixels_func (*op_pix)[4];
2328         qpel_mc_func (*op_qpix)[16];
2329         uint8_t *dest_y, *dest_cb, *dest_cr;
2330
2331         dest_y  = s->dest[0];
2332         dest_cb = s->dest[1];
2333         dest_cr = s->dest[2];
2334
2335         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
2336             op_pix  = s->hdsp.put_pixels_tab;
2337             op_qpix = s->qdsp.put_qpel_pixels_tab;
2338         } else {
2339             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
2340             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
2341         }
2342
2343         if (s->mv_dir & MV_DIR_FORWARD) {
2344             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 0,
2345                           s->last_picture.f->data,
2346                           op_pix, op_qpix);
2347             op_pix  = s->hdsp.avg_pixels_tab;
2348             op_qpix = s->qdsp.avg_qpel_pixels_tab;
2349         }
2350         if (s->mv_dir & MV_DIR_BACKWARD) {
2351             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 1,
2352                           s->next_picture.f->data,
2353                           op_pix, op_qpix);
2354         }
2355
2356         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2357             int progressive_score, interlaced_score;
2358
2359             s->interlaced_dct = 0;
2360             progressive_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y, wrap_y, 8) +
2361                                 s->mecc.ildct_cmp[0](s, dest_y + wrap_y * 8,
2362                                                      ptr_y + wrap_y * 8,
2363                                                      wrap_y, 8) - 400;
2364
2365             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2366                 progressive_score -= 400;
2367
2368             if (progressive_score > 0) {
2369                 interlaced_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y,
2370                                                         wrap_y * 2, 8) +
2371                                    s->mecc.ildct_cmp[0](s, dest_y + wrap_y,
2372                                                         ptr_y + wrap_y,
2373                                                         wrap_y * 2, 8);
2374
2375                 if (progressive_score > interlaced_score) {
2376                     s->interlaced_dct = 1;
2377
2378                     dct_offset = wrap_y;
2379                     uv_dct_offset = wrap_c;
2380                     wrap_y <<= 1;
2381                     if (s->chroma_format == CHROMA_422)
2382                         wrap_c <<= 1;
2383                 }
2384             }
2385         }
2386
2387         s->pdsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2388         s->pdsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2389         s->pdsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2390                             dest_y + dct_offset, wrap_y);
2391         s->pdsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2392                             dest_y + dct_offset + 8, wrap_y);
2393
2394         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2395             skip_dct[4] = 1;
2396             skip_dct[5] = 1;
2397         } else {
2398             s->pdsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2399             s->pdsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2400             if (!s->chroma_y_shift) { /* 422 */
2401                 s->pdsp.diff_pixels(s->block[6], ptr_cb + uv_dct_offset,
2402                                     dest_cb + uv_dct_offset, wrap_c);
2403                 s->pdsp.diff_pixels(s->block[7], ptr_cr + uv_dct_offset,
2404                                     dest_cr + uv_dct_offset, wrap_c);
2405             }
2406         }
2407         /* pre quantization */
2408         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2409                 2 * s->qscale * s->qscale) {
2410             // FIXME optimize
2411             if (s->mecc.sad[1](NULL, ptr_y, dest_y, wrap_y, 8) < 20 * s->qscale)
2412                 skip_dct[0] = 1;
2413             if (s->mecc.sad[1](NULL, ptr_y + 8, dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2414                 skip_dct[1] = 1;
2415             if (s->mecc.sad[1](NULL, ptr_y + dct_offset, dest_y + dct_offset,
2416                                wrap_y, 8) < 20 * s->qscale)
2417                 skip_dct[2] = 1;
2418             if (s->mecc.sad[1](NULL, ptr_y + dct_offset + 8, dest_y + dct_offset + 8,
2419                                wrap_y, 8) < 20 * s->qscale)
2420                 skip_dct[3] = 1;
2421             if (s->mecc.sad[1](NULL, ptr_cb, dest_cb, wrap_c, 8) < 20 * s->qscale)
2422                 skip_dct[4] = 1;
2423             if (s->mecc.sad[1](NULL, ptr_cr, dest_cr, wrap_c, 8) < 20 * s->qscale)
2424                 skip_dct[5] = 1;
2425             if (!s->chroma_y_shift) { /* 422 */
2426                 if (s->mecc.sad[1](NULL, ptr_cb + uv_dct_offset,
2427                                    dest_cb + uv_dct_offset,
2428                                    wrap_c, 8) < 20 * s->qscale)
2429                     skip_dct[6] = 1;
2430                 if (s->mecc.sad[1](NULL, ptr_cr + uv_dct_offset,
2431                                    dest_cr + uv_dct_offset,
2432                                    wrap_c, 8) < 20 * s->qscale)
2433                     skip_dct[7] = 1;
2434             }
2435         }
2436     }
2437
2438     if (s->quantizer_noise_shaping) {
2439         if (!skip_dct[0])
2440             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2441         if (!skip_dct[1])
2442             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2443         if (!skip_dct[2])
2444             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2445         if (!skip_dct[3])
2446             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2447         if (!skip_dct[4])
2448             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2449         if (!skip_dct[5])
2450             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2451         if (!s->chroma_y_shift) { /* 422 */
2452             if (!skip_dct[6])
2453                 get_visual_weight(weight[6], ptr_cb + uv_dct_offset,
2454                                   wrap_c);
2455             if (!skip_dct[7])
2456                 get_visual_weight(weight[7], ptr_cr + uv_dct_offset,
2457                                   wrap_c);
2458         }
2459         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2460     }
2461
2462     /* DCT & quantize */
2463     av_assert2(s->out_format != FMT_MJPEG || s->qscale == 8);
2464     {
2465         for (i = 0; i < mb_block_count; i++) {
2466             if (!skip_dct[i]) {
2467                 int overflow;
2468                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2469                 // FIXME we could decide to change to quantizer instead of
2470                 // clipping
2471                 // JS: I don't think that would be a good idea it could lower
2472                 //     quality instead of improve it. Just INTRADC clipping
2473                 //     deserves changes in quantizer
2474                 if (overflow)
2475                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2476             } else
2477                 s->block_last_index[i] = -1;
2478         }
2479         if (s->quantizer_noise_shaping) {
2480             for (i = 0; i < mb_block_count; i++) {
2481                 if (!skip_dct[i]) {
2482                     s->block_last_index[i] =
2483                         dct_quantize_refine(s, s->block[i], weight[i],
2484                                             orig[i], i, s->qscale);
2485                 }
2486             }
2487         }
2488
2489         if (s->luma_elim_threshold && !s->mb_intra)
2490             for (i = 0; i < 4; i++)
2491                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2492         if (s->chroma_elim_threshold && !s->mb_intra)
2493             for (i = 4; i < mb_block_count; i++)
2494                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2495
2496         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2497             for (i = 0; i < mb_block_count; i++) {
2498                 if (s->block_last_index[i] == -1)
2499                     s->coded_score[i] = INT_MAX / 256;
2500             }
2501         }
2502     }
2503
2504     if ((s->avctx->flags & AV_CODEC_FLAG_GRAY) && s->mb_intra) {
2505         s->block_last_index[4] =
2506         s->block_last_index[5] = 0;
2507         s->block[4][0] =
2508         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2509         if (!s->chroma_y_shift) { /* 422 / 444 */
2510             for (i=6; i<12; i++) {
2511                 s->block_last_index[i] = 0;
2512                 s->block[i][0] = s->block[4][0];
2513             }
2514         }
2515     }
2516
2517     // non c quantize code returns incorrect block_last_index FIXME
2518     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2519         for (i = 0; i < mb_block_count; i++) {
2520             int j;
2521             if (s->block_last_index[i] > 0) {
2522                 for (j = 63; j > 0; j--) {
2523                     if (s->block[i][s->intra_scantable.permutated[j]])
2524                         break;
2525                 }
2526                 s->block_last_index[i] = j;
2527             }
2528         }
2529     }
2530
2531     /* huffman encode */
2532     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2533     case AV_CODEC_ID_MPEG1VIDEO:
2534     case AV_CODEC_ID_MPEG2VIDEO:
2535         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2536             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2537         break;
2538     case AV_CODEC_ID_MPEG4:
2539         if (CONFIG_MPEG4_ENCODER)
2540             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2541         break;
2542     case AV_CODEC_ID_MSMPEG4V2:
2543     case AV_CODEC_ID_MSMPEG4V3:
2544     case AV_CODEC_ID_WMV1:
2545         if (CONFIG_MSMPEG4_ENCODER)
2546             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2547         break;
2548     case AV_CODEC_ID_WMV2:
2549         if (CONFIG_WMV2_ENCODER)
2550             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2551         break;
2552     case AV_CODEC_ID_H261:
2553         if (CONFIG_H261_ENCODER)
2554             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2555         break;
2556     case AV_CODEC_ID_H263:
2557     case AV_CODEC_ID_H263P:
2558     case AV_CODEC_ID_FLV1:
2559     case AV_CODEC_ID_RV10:
2560     case AV_CODEC_ID_RV20:
2561         if (CONFIG_H263_ENCODER)
2562             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2563         break;
2564     case AV_CODEC_ID_MJPEG:
2565     case AV_CODEC_ID_AMV:
2566         if (CONFIG_MJPEG_ENCODER)
2567             ff_mjpeg_encode_mb(s, s->block);
2568         break;
2569     default:
2570         av_assert1(0);
2571     }
2572 }
2573
2574 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2575 {
2576     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 8, 6);
2577     else if (s->chroma_format == CHROMA_422) encode_mb_internal(s, motion_x, motion_y, 16, 8, 8);
2578     else encode_mb_internal(s, motion_x, motion_y, 16, 16, 12);
2579 }
2580
2581 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2582     int i;
2583
2584     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2585
2586     /* MPEG-1 */
2587     d->mb_skip_run= s->mb_skip_run;
2588     for(i=0; i<3; i++)
2589         d->last_dc[i] = s->last_dc[i];
2590
2591     /* statistics */
2592     d->mv_bits= s->mv_bits;
2593     d->i_tex_bits= s->i_tex_bits;
2594     d->p_tex_bits= s->p_tex_bits;
2595     d->i_count= s->i_count;
2596     d->f_count= s->f_count;
2597     d->b_count= s->b_count;
2598     d->skip_count= s->skip_count;
2599     d->misc_bits= s->misc_bits;
2600     d->last_bits= 0;
2601
2602     d->mb_skipped= 0;
2603     d->qscale= s->qscale;
2604     d->dquant= s->dquant;
2605
2606     d->esc3_level_length= s->esc3_level_length;
2607 }
2608
2609 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2610     int i;
2611
2612     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2613     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2614
2615     /* MPEG-1 */
2616     d->mb_skip_run= s->mb_skip_run;
2617     for(i=0; i<3; i++)
2618         d->last_dc[i] = s->last_dc[i];
2619
2620     /* statistics */
2621     d->mv_bits= s->mv_bits;
2622     d->i_tex_bits= s->i_tex_bits;
2623     d->p_tex_bits= s->p_tex_bits;
2624     d->i_count= s->i_count;
2625     d->f_count= s->f_count;
2626     d->b_count= s->b_count;
2627     d->skip_count= s->skip_count;
2628     d->misc_bits= s->misc_bits;
2629
2630     d->mb_intra= s->mb_intra;
2631     d->mb_skipped= s->mb_skipped;
2632     d->mv_type= s->mv_type;
2633     d->mv_dir= s->mv_dir;
2634     d->pb= s->pb;
2635     if(s->data_partitioning){
2636         d->pb2= s->pb2;
2637         d->tex_pb= s->tex_pb;
2638     }
2639     d->block= s->block;
2640     for(i=0; i<8; i++)
2641         d->block_last_index[i]= s->block_last_index[i];
2642     d->interlaced_dct= s->interlaced_dct;
2643     d->qscale= s->qscale;
2644
2645     d->esc3_level_length= s->esc3_level_length;
2646 }
2647
2648 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2649                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2650                            int *dmin, int *next_block, int motion_x, int motion_y)
2651 {
2652     int score;
2653     uint8_t *dest_backup[3];
2654
2655     copy_context_before_encode(s, backup, type);
2656
2657     s->block= s->blocks[*next_block];
2658     s->pb= pb[*next_block];
2659     if(s->data_partitioning){
2660         s->pb2   = pb2   [*next_block];
2661         s->tex_pb= tex_pb[*next_block];
2662     }
2663
2664     if(*next_block){
2665         memcpy(dest_backup, s->dest, sizeof(s->dest));
2666         s->dest[0] = s->sc.rd_scratchpad;
2667         s->dest[1] = s->sc.rd_scratchpad + 16*s->linesize;
2668         s->dest[2] = s->sc.rd_scratchpad + 16*s->linesize + 8;
2669         av_assert0(s->linesize >= 32); //FIXME
2670     }
2671
2672     encode_mb(s, motion_x, motion_y);
2673
2674     score= put_bits_count(&s->pb);
2675     if(s->data_partitioning){
2676         score+= put_bits_count(&s->pb2);
2677         score+= put_bits_count(&s->tex_pb);
2678     }
2679
2680     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2681         ff_mpv_reconstruct_mb(s, s->block);
2682
2683         score *= s->lambda2;
2684         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2685     }
2686
2687     if(*next_block){
2688         memcpy(s->dest, dest_backup, sizeof(s->dest));
2689     }
2690
2691     if(score<*dmin){
2692         *dmin= score;
2693         *next_block^=1;
2694
2695         copy_context_after_encode(best, s, type);
2696     }
2697 }
2698
2699 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2700     const uint32_t *sq = ff_square_tab + 256;
2701     int acc=0;
2702     int x,y;
2703
2704     if(w==16 && h==16)
2705         return s->mecc.sse[0](NULL, src1, src2, stride, 16);
2706     else if(w==8 && h==8)
2707         return s->mecc.sse[1](NULL, src1, src2, stride, 8);
2708
2709     for(y=0; y<h; y++){
2710         for(x=0; x<w; x++){
2711             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2712         }
2713     }
2714
2715     av_assert2(acc>=0);
2716
2717     return acc;
2718 }
2719
2720 static int sse_mb(MpegEncContext *s){
2721     int w= 16;
2722     int h= 16;
2723
2724     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2725     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2726
2727     if(w==16 && h==16)
2728       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2729         return s->mecc.nsse[0](s, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2730                s->mecc.nsse[1](s, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2731                s->mecc.nsse[1](s, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2732       }else{
2733         return s->mecc.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2734                s->mecc.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2735                s->mecc.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2736       }
2737     else
2738         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2739                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2740                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2741 }
2742
2743 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2744     MpegEncContext *s= *(void**)arg;
2745
2746
2747     s->me.pre_pass=1;
2748     s->me.dia_size= s->avctx->pre_dia_size;
2749     s->first_slice_line=1;
2750     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2751         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2752             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2753         }
2754         s->first_slice_line=0;
2755     }
2756
2757     s->me.pre_pass=0;
2758
2759     return 0;
2760 }
2761
2762 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2763     MpegEncContext *s= *(void**)arg;
2764
2765     ff_check_alignment();
2766
2767     s->me.dia_size= s->avctx->dia_size;
2768     s->first_slice_line=1;
2769     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2770         s->mb_x=0; //for block init below
2771         ff_init_block_index(s);
2772         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2773             s->block_index[0]+=2;
2774             s->block_index[1]+=2;
2775             s->block_index[2]+=2;
2776             s->block_index[3]+=2;
2777
2778             /* compute motion vector & mb_type and store in context */
2779             if(s->pict_type==AV_PICTURE_TYPE_B)
2780                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2781             else
2782                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2783         }
2784         s->first_slice_line=0;
2785     }
2786     return 0;
2787 }
2788
2789 static int mb_var_thread(AVCodecContext *c, void *arg){
2790     MpegEncContext *s= *(void**)arg;
2791     int mb_x, mb_y;
2792
2793     ff_check_alignment();
2794
2795     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2796         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2797             int xx = mb_x * 16;
2798             int yy = mb_y * 16;
2799             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2800             int varc;
2801             int sum = s->mpvencdsp.pix_sum(pix, s->linesize);
2802
2803             varc = (s->mpvencdsp.pix_norm1(pix, s->linesize) -
2804                     (((unsigned) sum * sum) >> 8) + 500 + 128) >> 8;
2805
2806             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2807             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2808             s->me.mb_var_sum_temp    += varc;
2809         }
2810     }
2811     return 0;
2812 }
2813
2814 static void write_slice_end(MpegEncContext *s){
2815     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2816         if(s->partitioned_frame){
2817             ff_mpeg4_merge_partitions(s);
2818         }
2819
2820         ff_mpeg4_stuffing(&s->pb);
2821     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2822         ff_mjpeg_encode_stuffing(s);
2823     }
2824
2825     avpriv_align_put_bits(&s->pb);
2826     flush_put_bits(&s->pb);
2827
2828     if ((s->avctx->flags & AV_CODEC_FLAG_PASS1) && !s->partitioned_frame)
2829         s->misc_bits+= get_bits_diff(s);
2830 }
2831
2832 static void write_mb_info(MpegEncContext *s)
2833 {
2834     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2835     int offset = put_bits_count(&s->pb);
2836     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2837     int gobn = s->mb_y / s->gob_index;
2838     int pred_x, pred_y;
2839     if (CONFIG_H263_ENCODER)
2840         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2841     bytestream_put_le32(&ptr, offset);
2842     bytestream_put_byte(&ptr, s->qscale);
2843     bytestream_put_byte(&ptr, gobn);
2844     bytestream_put_le16(&ptr, mba);
2845     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2846     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2847     /* 4MV not implemented */
2848     bytestream_put_byte(&ptr, 0); /* hmv2 */
2849     bytestream_put_byte(&ptr, 0); /* vmv2 */
2850 }
2851
2852 static void update_mb_info(MpegEncContext *s, int startcode)
2853 {
2854     if (!s->mb_info)
2855         return;
2856     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2857         s->mb_info_size += 12;
2858         s->prev_mb_info = s->last_mb_info;
2859     }
2860     if (startcode) {
2861         s->prev_mb_info = put_bits_count(&s->pb)/8;
2862         /* This might have incremented mb_info_size above, and we return without
2863          * actually writing any info into that slot yet. But in that case,
2864          * this will be called again at the start of the after writing the
2865          * start code, actually writing the mb info. */
2866         return;
2867     }
2868
2869     s->last_mb_info = put_bits_count(&s->pb)/8;
2870     if (!s->mb_info_size)
2871         s->mb_info_size += 12;
2872     write_mb_info(s);
2873 }
2874
2875 int ff_mpv_reallocate_putbitbuffer(MpegEncContext *s, size_t threshold, size_t size_increase)
2876 {
2877     if (   s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < threshold
2878         && s->slice_context_count == 1
2879         && s->pb.buf == s->avctx->internal->byte_buffer) {
2880         int lastgob_pos = s->ptr_lastgob - s->pb.buf;
2881         int vbv_pos     = s->vbv_delay_ptr - s->pb.buf;
2882
2883         uint8_t *new_buffer = NULL;
2884         int new_buffer_size = 0;
2885
2886         if ((s->avctx->internal->byte_buffer_size + size_increase) >= INT_MAX/8) {
2887             av_log(s->avctx, AV_LOG_ERROR, "Cannot reallocate putbit buffer\n");
2888             return AVERROR(ENOMEM);
2889         }
2890
2891         emms_c();
2892
2893         av_fast_padded_malloc(&new_buffer, &new_buffer_size,
2894                               s->avctx->internal->byte_buffer_size + size_increase);
2895         if (!new_buffer)
2896             return AVERROR(ENOMEM);
2897
2898         memcpy(new_buffer, s->avctx->internal->byte_buffer, s->avctx->internal->byte_buffer_size);
2899         av_free(s->avctx->internal->byte_buffer);
2900         s->avctx->internal->byte_buffer      = new_buffer;
2901         s->avctx->internal->byte_buffer_size = new_buffer_size;
2902         rebase_put_bits(&s->pb, new_buffer, new_buffer_size);
2903         s->ptr_lastgob   = s->pb.buf + lastgob_pos;
2904         s->vbv_delay_ptr = s->pb.buf + vbv_pos;
2905     }
2906     if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < threshold)
2907         return AVERROR(EINVAL);
2908     return 0;
2909 }
2910
2911 static int encode_thread(AVCodecContext *c, void *arg){
2912     MpegEncContext *s= *(void**)arg;
2913     int mb_x, mb_y;
2914     int chr_h= 16>>s->chroma_y_shift;
2915     int i, j;
2916     MpegEncContext best_s = { 0 }, backup_s;
2917     uint8_t bit_buf[2][MAX_MB_BYTES];
2918     uint8_t bit_buf2[2][MAX_MB_BYTES];
2919     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2920     PutBitContext pb[2], pb2[2], tex_pb[2];
2921
2922     ff_check_alignment();
2923
2924     for(i=0; i<2; i++){
2925         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2926         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2927         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2928     }
2929
2930     s->last_bits= put_bits_count(&s->pb);
2931     s->mv_bits=0;
2932     s->misc_bits=0;
2933     s->i_tex_bits=0;
2934     s->p_tex_bits=0;
2935     s->i_count=0;
2936     s->f_count=0;
2937     s->b_count=0;
2938     s->skip_count=0;
2939
2940     for(i=0; i<3; i++){
2941         /* init last dc values */
2942         /* note: quant matrix value (8) is implied here */
2943         s->last_dc[i] = 128 << s->intra_dc_precision;
2944
2945         s->current_picture.encoding_error[i] = 0;
2946     }
2947     if(s->codec_id==AV_CODEC_ID_AMV){
2948         s->last_dc[0] = 128*8/13;
2949         s->last_dc[1] = 128*8/14;
2950         s->last_dc[2] = 128*8/14;
2951     }
2952     s->mb_skip_run = 0;
2953     memset(s->last_mv, 0, sizeof(s->last_mv));
2954
2955     s->last_mv_dir = 0;
2956
2957     switch(s->codec_id){
2958     case AV_CODEC_ID_H263:
2959     case AV_CODEC_ID_H263P:
2960     case AV_CODEC_ID_FLV1:
2961         if (CONFIG_H263_ENCODER)
2962             s->gob_index = H263_GOB_HEIGHT(s->height);
2963         break;
2964     case AV_CODEC_ID_MPEG4:
2965         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2966             ff_mpeg4_init_partitions(s);
2967         break;
2968     }
2969
2970     s->resync_mb_x=0;
2971     s->resync_mb_y=0;
2972     s->first_slice_line = 1;
2973     s->ptr_lastgob = s->pb.buf;
2974     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2975         s->mb_x=0;
2976         s->mb_y= mb_y;
2977
2978         ff_set_qscale(s, s->qscale);
2979         ff_init_block_index(s);
2980
2981         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2982             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2983             int mb_type= s->mb_type[xy];
2984 //            int d;
2985             int dmin= INT_MAX;
2986             int dir;
2987             int size_increase =  s->avctx->internal->byte_buffer_size/4
2988                                + s->mb_width*MAX_MB_BYTES;
2989
2990             ff_mpv_reallocate_putbitbuffer(s, MAX_MB_BYTES, size_increase);
2991             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2992                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2993                 return -1;
2994             }
2995             if(s->data_partitioning){
2996                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2997                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2998                     av_log(s->avctx, AV_LOG_ERROR, "encoded partitioned frame too large\n");
2999                     return -1;
3000                 }
3001             }
3002
3003             s->mb_x = mb_x;
3004             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
3005             ff_update_block_index(s);
3006
3007             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
3008                 ff_h261_reorder_mb_index(s);
3009                 xy= s->mb_y*s->mb_stride + s->mb_x;
3010                 mb_type= s->mb_type[xy];
3011             }
3012
3013             /* write gob / video packet header  */
3014             if(s->rtp_mode){
3015                 int current_packet_size, is_gob_start;
3016
3017                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
3018
3019                 is_gob_start = s->rtp_payload_size &&
3020                                current_packet_size >= s->rtp_payload_size &&
3021                                mb_y + mb_x > 0;
3022
3023                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
3024
3025                 switch(s->codec_id){
3026                 case AV_CODEC_ID_H263:
3027                 case AV_CODEC_ID_H263P:
3028                     if(!s->h263_slice_structured)
3029                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
3030                     break;
3031                 case AV_CODEC_ID_MPEG2VIDEO:
3032                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
3033                 case AV_CODEC_ID_MPEG1VIDEO:
3034                     if(s->mb_skip_run) is_gob_start=0;
3035                     break;
3036                 case AV_CODEC_ID_MJPEG:
3037                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
3038                     break;
3039                 }
3040
3041                 if(is_gob_start){
3042                     if(s->start_mb_y != mb_y || mb_x!=0){
3043                         write_slice_end(s);
3044
3045                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
3046                             ff_mpeg4_init_partitions(s);
3047                         }
3048                     }
3049
3050                     av_assert2((put_bits_count(&s->pb)&7) == 0);
3051                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
3052
3053                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
3054                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
3055                         int d = 100 / s->error_rate;
3056                         if(r % d == 0){
3057                             current_packet_size=0;
3058                             s->pb.buf_ptr= s->ptr_lastgob;
3059                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
3060                         }
3061                     }
3062
3063 #if FF_API_RTP_CALLBACK
3064 FF_DISABLE_DEPRECATION_WARNINGS
3065                     if (s->avctx->rtp_callback){
3066                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
3067                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
3068                     }
3069 FF_ENABLE_DEPRECATION_WARNINGS
3070 #endif
3071                     update_mb_info(s, 1);
3072
3073                     switch(s->codec_id){
3074                     case AV_CODEC_ID_MPEG4:
3075                         if (CONFIG_MPEG4_ENCODER) {
3076                             ff_mpeg4_encode_video_packet_header(s);
3077                             ff_mpeg4_clean_buffers(s);
3078                         }
3079                     break;
3080                     case AV_CODEC_ID_MPEG1VIDEO:
3081                     case AV_CODEC_ID_MPEG2VIDEO:
3082                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
3083                             ff_mpeg1_encode_slice_header(s);
3084                             ff_mpeg1_clean_buffers(s);
3085                         }
3086                     break;
3087                     case AV_CODEC_ID_H263:
3088                     case AV_CODEC_ID_H263P:
3089                         if (CONFIG_H263_ENCODER)
3090                             ff_h263_encode_gob_header(s, mb_y);
3091                     break;
3092                     }
3093
3094                     if (s->avctx->flags & AV_CODEC_FLAG_PASS1) {
3095                         int bits= put_bits_count(&s->pb);
3096                         s->misc_bits+= bits - s->last_bits;
3097                         s->last_bits= bits;
3098                     }
3099
3100                     s->ptr_lastgob += current_packet_size;
3101                     s->first_slice_line=1;
3102                     s->resync_mb_x=mb_x;
3103                     s->resync_mb_y=mb_y;
3104                 }
3105             }
3106
3107             if(  (s->resync_mb_x   == s->mb_x)
3108                && s->resync_mb_y+1 == s->mb_y){
3109                 s->first_slice_line=0;
3110             }
3111
3112             s->mb_skipped=0;
3113             s->dquant=0; //only for QP_RD
3114
3115             update_mb_info(s, 0);
3116
3117             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
3118                 int next_block=0;
3119                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
3120
3121                 copy_context_before_encode(&backup_s, s, -1);
3122                 backup_s.pb= s->pb;
3123                 best_s.data_partitioning= s->data_partitioning;
3124                 best_s.partitioned_frame= s->partitioned_frame;
3125                 if(s->data_partitioning){
3126                     backup_s.pb2= s->pb2;
3127                     backup_s.tex_pb= s->tex_pb;
3128                 }
3129
3130                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
3131                     s->mv_dir = MV_DIR_FORWARD;
3132                     s->mv_type = MV_TYPE_16X16;
3133                     s->mb_intra= 0;
3134                     s->mv[0][0][0] = s->p_mv_table[xy][0];
3135                     s->mv[0][0][1] = s->p_mv_table[xy][1];
3136                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
3137                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
3138                 }
3139                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
3140                     s->mv_dir = MV_DIR_FORWARD;
3141                     s->mv_type = MV_TYPE_FIELD;
3142                     s->mb_intra= 0;
3143                     for(i=0; i<2; i++){
3144                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3145                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3146                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3147                     }
3148                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
3149                                  &dmin, &next_block, 0, 0);
3150                 }
3151                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
3152                     s->mv_dir = MV_DIR_FORWARD;
3153                     s->mv_type = MV_TYPE_16X16;
3154                     s->mb_intra= 0;
3155                     s->mv[0][0][0] = 0;
3156                     s->mv[0][0][1] = 0;
3157                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
3158                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
3159                 }
3160                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
3161                     s->mv_dir = MV_DIR_FORWARD;
3162                     s->mv_type = MV_TYPE_8X8;
3163                     s->mb_intra= 0;
3164                     for(i=0; i<4; i++){
3165                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3166                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3167                     }
3168                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
3169                                  &dmin, &next_block, 0, 0);
3170                 }
3171                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
3172                     s->mv_dir = MV_DIR_FORWARD;
3173                     s->mv_type = MV_TYPE_16X16;
3174                     s->mb_intra= 0;
3175                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3176                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3177                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
3178                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
3179                 }
3180                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
3181                     s->mv_dir = MV_DIR_BACKWARD;
3182                     s->mv_type = MV_TYPE_16X16;
3183                     s->mb_intra= 0;
3184                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3185                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3186                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
3187                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
3188                 }
3189                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
3190                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3191                     s->mv_type = MV_TYPE_16X16;
3192                     s->mb_intra= 0;
3193                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3194                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3195                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3196                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3197                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
3198                                  &dmin, &next_block, 0, 0);
3199                 }
3200                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
3201                     s->mv_dir = MV_DIR_FORWARD;
3202                     s->mv_type = MV_TYPE_FIELD;
3203                     s->mb_intra= 0;
3204                     for(i=0; i<2; i++){
3205                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3206                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3207                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3208                     }
3209                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
3210                                  &dmin, &next_block, 0, 0);
3211                 }
3212                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
3213                     s->mv_dir = MV_DIR_BACKWARD;
3214                     s->mv_type = MV_TYPE_FIELD;
3215                     s->mb_intra= 0;
3216                     for(i=0; i<2; i++){
3217                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3218                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3219                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3220                     }
3221                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
3222                                  &dmin, &next_block, 0, 0);
3223                 }
3224                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
3225                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3226                     s->mv_type = MV_TYPE_FIELD;
3227                     s->mb_intra= 0;
3228                     for(dir=0; dir<2; dir++){
3229                         for(i=0; i<2; i++){
3230                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3231                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3232                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3233                         }
3234                     }
3235                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
3236                                  &dmin, &next_block, 0, 0);
3237                 }
3238                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
3239                     s->mv_dir = 0;
3240                     s->mv_type = MV_TYPE_16X16;
3241                     s->mb_intra= 1;
3242                     s->mv[0][0][0] = 0;
3243                     s->mv[0][0][1] = 0;
3244                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
3245                                  &dmin, &next_block, 0, 0);
3246                     if(s->h263_pred || s->h263_aic){
3247                         if(best_s.mb_intra)
3248                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
3249                         else
3250                             ff_clean_intra_table_entries(s); //old mode?
3251                     }
3252                 }
3253
3254                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
3255                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
3256                         const int last_qp= backup_s.qscale;
3257                         int qpi, qp, dc[6];
3258                         int16_t ac[6][16];
3259                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
3260                         static const int dquant_tab[4]={-1,1,-2,2};
3261                         int storecoefs = s->mb_intra && s->dc_val[0];
3262
3263                         av_assert2(backup_s.dquant == 0);
3264
3265                         //FIXME intra
3266                         s->mv_dir= best_s.mv_dir;
3267                         s->mv_type = MV_TYPE_16X16;
3268                         s->mb_intra= best_s.mb_intra;
3269                         s->mv[0][0][0] = best_s.mv[0][0][0];
3270                         s->mv[0][0][1] = best_s.mv[0][0][1];
3271                         s->mv[1][0][0] = best_s.mv[1][0][0];
3272                         s->mv[1][0][1] = best_s.mv[1][0][1];
3273
3274                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
3275                         for(; qpi<4; qpi++){
3276                             int dquant= dquant_tab[qpi];
3277                             qp= last_qp + dquant;
3278                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
3279                                 continue;
3280                             backup_s.dquant= dquant;
3281                             if(storecoefs){
3282                                 for(i=0; i<6; i++){
3283                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
3284                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(int16_t)*16);
3285                                 }
3286                             }
3287
3288                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3289                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
3290                             if(best_s.qscale != qp){
3291                                 if(storecoefs){
3292                                     for(i=0; i<6; i++){
3293                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
3294                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(int16_t)*16);
3295                                     }
3296                                 }
3297                             }
3298                         }
3299                     }
3300                 }
3301                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
3302                     int mx= s->b_direct_mv_table[xy][0];
3303                     int my= s->b_direct_mv_table[xy][1];
3304
3305                     backup_s.dquant = 0;
3306                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3307                     s->mb_intra= 0;
3308                     ff_mpeg4_set_direct_mv(s, mx, my);
3309                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3310                                  &dmin, &next_block, mx, my);
3311                 }
3312                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
3313                     backup_s.dquant = 0;
3314                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3315                     s->mb_intra= 0;
3316                     ff_mpeg4_set_direct_mv(s, 0, 0);
3317                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3318                                  &dmin, &next_block, 0, 0);
3319                 }
3320                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
3321                     int coded=0;
3322                     for(i=0; i<6; i++)
3323                         coded |= s->block_last_index[i];
3324                     if(coded){
3325                         int mx,my;
3326                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
3327                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
3328                             mx=my=0; //FIXME find the one we actually used
3329                             ff_mpeg4_set_direct_mv(s, mx, my);
3330                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
3331                             mx= s->mv[1][0][0];