15cca64c492d62e15a390dba5529fb6ef06fa7d5
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of Libav.
9  *
10  * Libav is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * Libav is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with Libav; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include <stdint.h>
31
32 #include "libavutil/internal.h"
33 #include "libavutil/intmath.h"
34 #include "libavutil/mathematics.h"
35 #include "libavutil/pixdesc.h"
36 #include "libavutil/opt.h"
37 #include "libavutil/timer.h"
38 #include "avcodec.h"
39 #include "dct.h"
40 #include "idctdsp.h"
41 #include "mpeg12.h"
42 #include "mpegvideo.h"
43 #include "h261.h"
44 #include "h263.h"
45 #include "mjpegenc_common.h"
46 #include "mathops.h"
47 #include "mpegutils.h"
48 #include "mjpegenc.h"
49 #include "msmpeg4.h"
50 #include "pixblockdsp.h"
51 #include "qpeldsp.h"
52 #include "faandct.h"
53 #include "thread.h"
54 #include "aandcttab.h"
55 #include "flv.h"
56 #include "mpeg4video.h"
57 #include "internal.h"
58 #include "bytestream.h"
59 #include <limits.h>
60
61 #define QUANT_BIAS_SHIFT 8
62
63 #define QMAT_SHIFT_MMX 16
64 #define QMAT_SHIFT 22
65
66 static int encode_picture(MpegEncContext *s, int picture_number);
67 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
68 static int sse_mb(MpegEncContext *s);
69 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
70 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
71
72 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
73 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
74
75 const AVOption ff_mpv_generic_options[] = {
76     FF_MPV_COMMON_OPTS
77     { NULL },
78 };
79
80 void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64],
81                        uint16_t (*qmat16)[2][64],
82                        const uint16_t *quant_matrix,
83                        int bias, int qmin, int qmax, int intra)
84 {
85     FDCTDSPContext *fdsp = &s->fdsp;
86     int qscale;
87     int shift = 0;
88
89     for (qscale = qmin; qscale <= qmax; qscale++) {
90         int i;
91         if (fdsp->fdct == ff_jpeg_fdct_islow_8  ||
92 #if CONFIG_FAANDCT
93             fdsp->fdct == ff_faandct            ||
94 #endif /* CONFIG_FAANDCT */
95             fdsp->fdct == ff_jpeg_fdct_islow_10) {
96             for (i = 0; i < 64; i++) {
97                 const int j = s->idsp.idct_permutation[i];
98                 int64_t den = (int64_t) qscale * quant_matrix[j];
99                 /* 16 <= qscale * quant_matrix[i] <= 7905
100                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
101                  *             19952 <=              x  <= 249205026
102                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
103                  *           3444240 >= (1 << 36) / (x) >= 275 */
104
105                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
106             }
107         } else if (fdsp->fdct == ff_fdct_ifast) {
108             for (i = 0; i < 64; i++) {
109                 const int j = s->idsp.idct_permutation[i];
110                 int64_t den = ff_aanscales[i] * (int64_t) qscale * quant_matrix[j];
111                 /* 16 <= qscale * quant_matrix[i] <= 7905
112                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
113                  *             19952 <=              x  <= 249205026
114                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
115                  *           3444240 >= (1 << 36) / (x) >= 275 */
116
117                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / den);
118             }
119         } else {
120             for (i = 0; i < 64; i++) {
121                 const int j = s->idsp.idct_permutation[i];
122                 int64_t den = (int64_t) qscale * quant_matrix[j];
123                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
124                  * Assume x = qscale * quant_matrix[i]
125                  * So             16 <=              x  <= 7905
126                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
127                  * so          32768 >= (1 << 19) / (x) >= 67 */
128                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
129                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
130                 //                    (qscale * quant_matrix[i]);
131                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / den;
132
133                 if (qmat16[qscale][0][i] == 0 ||
134                     qmat16[qscale][0][i] == 128 * 256)
135                     qmat16[qscale][0][i] = 128 * 256 - 1;
136                 qmat16[qscale][1][i] =
137                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
138                                 qmat16[qscale][0][i]);
139             }
140         }
141
142         for (i = intra; i < 64; i++) {
143             int64_t max = 8191;
144             if (fdsp->fdct == ff_fdct_ifast) {
145                 max = (8191LL * ff_aanscales[i]) >> 14;
146             }
147             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
148                 shift++;
149             }
150         }
151     }
152     if (shift) {
153         av_log(NULL, AV_LOG_INFO,
154                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
155                QMAT_SHIFT - shift);
156     }
157 }
158
159 static inline void update_qscale(MpegEncContext *s)
160 {
161     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
162                 (FF_LAMBDA_SHIFT + 7);
163     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
164
165     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
166                  FF_LAMBDA_SHIFT;
167 }
168
169 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
170 {
171     int i;
172
173     if (matrix) {
174         put_bits(pb, 1, 1);
175         for (i = 0; i < 64; i++) {
176             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
177         }
178     } else
179         put_bits(pb, 1, 0);
180 }
181
182 /**
183  * init s->current_picture.qscale_table from s->lambda_table
184  */
185 void ff_init_qscale_tab(MpegEncContext *s)
186 {
187     int8_t * const qscale_table = s->current_picture.qscale_table;
188     int i;
189
190     for (i = 0; i < s->mb_num; i++) {
191         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
192         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
193         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
194                                                   s->avctx->qmax);
195     }
196 }
197
198 static void update_duplicate_context_after_me(MpegEncContext *dst,
199                                               MpegEncContext *src)
200 {
201 #define COPY(a) dst->a= src->a
202     COPY(pict_type);
203     COPY(current_picture);
204     COPY(f_code);
205     COPY(b_code);
206     COPY(qscale);
207     COPY(lambda);
208     COPY(lambda2);
209     COPY(picture_in_gop_number);
210     COPY(gop_picture_number);
211     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
212     COPY(progressive_frame);    // FIXME don't set in encode_header
213     COPY(partitioned_frame);    // FIXME don't set in encode_header
214 #undef COPY
215 }
216
217 /**
218  * Set the given MpegEncContext to defaults for encoding.
219  * the changed fields will not depend upon the prior state of the MpegEncContext.
220  */
221 static void mpv_encode_defaults(MpegEncContext *s)
222 {
223     int i;
224     ff_mpv_common_defaults(s);
225
226     for (i = -16; i < 16; i++) {
227         default_fcode_tab[i + MAX_MV] = 1;
228     }
229     s->me.mv_penalty = default_mv_penalty;
230     s->fcode_tab     = default_fcode_tab;
231
232     s->input_picture_number  = 0;
233     s->picture_in_gop_number = 0;
234 }
235
236 /* init video encoder */
237 av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
238 {
239     MpegEncContext *s = avctx->priv_data;
240     int i, ret, format_supported;
241
242     mpv_encode_defaults(s);
243
244     switch (avctx->codec_id) {
245     case AV_CODEC_ID_MPEG2VIDEO:
246         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
247             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
248             av_log(avctx, AV_LOG_ERROR,
249                    "only YUV420 and YUV422 are supported\n");
250             return -1;
251         }
252         break;
253     case AV_CODEC_ID_MJPEG:
254         format_supported = 0;
255         /* JPEG color space */
256         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
257             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
258             (avctx->color_range == AVCOL_RANGE_JPEG &&
259              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
260               avctx->pix_fmt == AV_PIX_FMT_YUV422P)))
261             format_supported = 1;
262         /* MPEG color space */
263         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
264                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
265                   avctx->pix_fmt == AV_PIX_FMT_YUV422P))
266             format_supported = 1;
267
268         if (!format_supported) {
269             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
270             return -1;
271         }
272         break;
273     default:
274         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
275             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
276             return -1;
277         }
278     }
279
280     switch (avctx->pix_fmt) {
281     case AV_PIX_FMT_YUVJ422P:
282     case AV_PIX_FMT_YUV422P:
283         s->chroma_format = CHROMA_422;
284         break;
285     case AV_PIX_FMT_YUVJ420P:
286     case AV_PIX_FMT_YUV420P:
287     default:
288         s->chroma_format = CHROMA_420;
289         break;
290     }
291
292     s->bit_rate = avctx->bit_rate;
293     s->width    = avctx->width;
294     s->height   = avctx->height;
295     if (avctx->gop_size > 600 &&
296         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
297         av_log(avctx, AV_LOG_ERROR,
298                "Warning keyframe interval too large! reducing it ...\n");
299         avctx->gop_size = 600;
300     }
301     s->gop_size     = avctx->gop_size;
302     s->avctx        = avctx;
303     if (avctx->max_b_frames > MAX_B_FRAMES) {
304         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
305                "is %d.\n", MAX_B_FRAMES);
306     }
307     s->max_b_frames = avctx->max_b_frames;
308     s->codec_id     = avctx->codec->id;
309     s->strict_std_compliance = avctx->strict_std_compliance;
310     s->quarter_sample     = (avctx->flags & CODEC_FLAG_QPEL) != 0;
311     s->mpeg_quant         = avctx->mpeg_quant;
312     s->rtp_mode           = !!avctx->rtp_payload_size;
313     s->intra_dc_precision = avctx->intra_dc_precision;
314     s->user_specified_pts = AV_NOPTS_VALUE;
315
316     if (s->gop_size <= 1) {
317         s->intra_only = 1;
318         s->gop_size   = 12;
319     } else {
320         s->intra_only = 0;
321     }
322
323     s->me_method = avctx->me_method;
324
325     /* Fixed QSCALE */
326     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
327
328 #if FF_API_MPV_OPT
329     FF_DISABLE_DEPRECATION_WARNINGS
330     if (avctx->border_masking != 0.0)
331         s->border_masking = avctx->border_masking;
332     FF_ENABLE_DEPRECATION_WARNINGS
333 #endif
334
335     s->adaptive_quant = (s->avctx->lumi_masking ||
336                          s->avctx->dark_masking ||
337                          s->avctx->temporal_cplx_masking ||
338                          s->avctx->spatial_cplx_masking  ||
339                          s->avctx->p_masking      ||
340                          s->border_masking ||
341                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
342                         !s->fixed_qscale;
343
344     s->loop_filter = !!(s->avctx->flags & CODEC_FLAG_LOOP_FILTER);
345
346     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
347         av_log(avctx, AV_LOG_ERROR,
348                "a vbv buffer size is needed, "
349                "for encoding with a maximum bitrate\n");
350         return -1;
351     }
352
353     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
354         av_log(avctx, AV_LOG_INFO,
355                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
356     }
357
358     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
359         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
360         return -1;
361     }
362
363     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
364         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
365         return -1;
366     }
367
368     if (avctx->rc_max_rate &&
369         avctx->rc_max_rate == avctx->bit_rate &&
370         avctx->rc_max_rate != avctx->rc_min_rate) {
371         av_log(avctx, AV_LOG_INFO,
372                "impossible bitrate constraints, this will fail\n");
373     }
374
375     if (avctx->rc_buffer_size &&
376         avctx->bit_rate * (int64_t)avctx->time_base.num >
377             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
378         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
379         return -1;
380     }
381
382     if (!s->fixed_qscale &&
383         avctx->bit_rate * av_q2d(avctx->time_base) >
384             avctx->bit_rate_tolerance) {
385         av_log(avctx, AV_LOG_ERROR,
386                "bitrate tolerance too small for bitrate\n");
387         return -1;
388     }
389
390     if (s->avctx->rc_max_rate &&
391         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
392         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
393          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
394         90000LL * (avctx->rc_buffer_size - 1) >
395             s->avctx->rc_max_rate * 0xFFFFLL) {
396         av_log(avctx, AV_LOG_INFO,
397                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
398                "specified vbv buffer is too large for the given bitrate!\n");
399     }
400
401     if ((s->avctx->flags & CODEC_FLAG_4MV) && s->codec_id != AV_CODEC_ID_MPEG4 &&
402         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
403         s->codec_id != AV_CODEC_ID_FLV1) {
404         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
405         return -1;
406     }
407
408     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
409         av_log(avctx, AV_LOG_ERROR,
410                "OBMC is only supported with simple mb decision\n");
411         return -1;
412     }
413
414     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
415         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
416         return -1;
417     }
418
419     if (s->max_b_frames                    &&
420         s->codec_id != AV_CODEC_ID_MPEG4      &&
421         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
422         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
423         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
424         return -1;
425     }
426
427     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
428          s->codec_id == AV_CODEC_ID_H263  ||
429          s->codec_id == AV_CODEC_ID_H263P) &&
430         (avctx->sample_aspect_ratio.num > 255 ||
431          avctx->sample_aspect_ratio.den > 255)) {
432         av_log(avctx, AV_LOG_ERROR,
433                "Invalid pixel aspect ratio %i/%i, limit is 255/255\n",
434                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
435         return -1;
436     }
437
438     if ((s->avctx->flags & (CODEC_FLAG_INTERLACED_DCT | CODEC_FLAG_INTERLACED_ME)) &&
439         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
440         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
441         return -1;
442     }
443
444     // FIXME mpeg2 uses that too
445     if (s->mpeg_quant && s->codec_id != AV_CODEC_ID_MPEG4) {
446         av_log(avctx, AV_LOG_ERROR,
447                "mpeg2 style quantization not supported by codec\n");
448         return -1;
449     }
450
451     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
452         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
453         return -1;
454     }
455
456     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
457         s->avctx->mb_decision != FF_MB_DECISION_RD) {
458         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
459         return -1;
460     }
461
462     if (s->avctx->scenechange_threshold < 1000000000 &&
463         (s->avctx->flags & CODEC_FLAG_CLOSED_GOP)) {
464         av_log(avctx, AV_LOG_ERROR,
465                "closed gop with scene change detection are not supported yet, "
466                "set threshold to 1000000000\n");
467         return -1;
468     }
469
470     if (s->avctx->flags & CODEC_FLAG_LOW_DELAY) {
471         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
472             av_log(avctx, AV_LOG_ERROR,
473                   "low delay forcing is only available for mpeg2\n");
474             return -1;
475         }
476         if (s->max_b_frames != 0) {
477             av_log(avctx, AV_LOG_ERROR,
478                    "b frames cannot be used with low delay\n");
479             return -1;
480         }
481     }
482
483     if (s->q_scale_type == 1) {
484         if (avctx->qmax > 12) {
485             av_log(avctx, AV_LOG_ERROR,
486                    "non linear quant only supports qmax <= 12 currently\n");
487             return -1;
488         }
489     }
490
491     if (s->avctx->thread_count > 1         &&
492         s->codec_id != AV_CODEC_ID_MPEG4      &&
493         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
494         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
495         (s->codec_id != AV_CODEC_ID_H263P)) {
496         av_log(avctx, AV_LOG_ERROR,
497                "multi threaded encoding not supported by codec\n");
498         return -1;
499     }
500
501     if (s->avctx->thread_count < 1) {
502         av_log(avctx, AV_LOG_ERROR,
503                "automatic thread number detection not supported by codec,"
504                "patch welcome\n");
505         return -1;
506     }
507
508     if (s->avctx->thread_count > 1)
509         s->rtp_mode = 1;
510
511     if (!avctx->time_base.den || !avctx->time_base.num) {
512         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
513         return -1;
514     }
515
516     if (avctx->b_frame_strategy && (avctx->flags & CODEC_FLAG_PASS2)) {
517         av_log(avctx, AV_LOG_INFO,
518                "notice: b_frame_strategy only affects the first pass\n");
519         avctx->b_frame_strategy = 0;
520     }
521
522     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
523     if (i > 1) {
524         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
525         avctx->time_base.den /= i;
526         avctx->time_base.num /= i;
527         //return -1;
528     }
529
530     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
531         s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG) {
532         // (a + x * 3 / 8) / x
533         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
534         s->inter_quant_bias = 0;
535     } else {
536         s->intra_quant_bias = 0;
537         // (a - x / 4) / x
538         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
539     }
540
541     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
542         s->intra_quant_bias = avctx->intra_quant_bias;
543     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
544         s->inter_quant_bias = avctx->inter_quant_bias;
545
546     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
547         s->avctx->time_base.den > (1 << 16) - 1) {
548         av_log(avctx, AV_LOG_ERROR,
549                "timebase %d/%d not supported by MPEG 4 standard, "
550                "the maximum admitted value for the timebase denominator "
551                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
552                (1 << 16) - 1);
553         return -1;
554     }
555     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
556
557     switch (avctx->codec->id) {
558     case AV_CODEC_ID_MPEG1VIDEO:
559         s->out_format = FMT_MPEG1;
560         s->low_delay  = !!(s->avctx->flags & CODEC_FLAG_LOW_DELAY);
561         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
562         break;
563     case AV_CODEC_ID_MPEG2VIDEO:
564         s->out_format = FMT_MPEG1;
565         s->low_delay  = !!(s->avctx->flags & CODEC_FLAG_LOW_DELAY);
566         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
567         s->rtp_mode   = 1;
568         break;
569     case AV_CODEC_ID_MJPEG:
570         s->out_format = FMT_MJPEG;
571         s->intra_only = 1; /* force intra only for jpeg */
572         if (!CONFIG_MJPEG_ENCODER ||
573             ff_mjpeg_encode_init(s) < 0)
574             return -1;
575         avctx->delay = 0;
576         s->low_delay = 1;
577         break;
578     case AV_CODEC_ID_H261:
579         if (!CONFIG_H261_ENCODER)
580             return -1;
581         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
582             av_log(avctx, AV_LOG_ERROR,
583                    "The specified picture size of %dx%d is not valid for the "
584                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
585                     s->width, s->height);
586             return -1;
587         }
588         s->out_format = FMT_H261;
589         avctx->delay  = 0;
590         s->low_delay  = 1;
591         s->rtp_mode   = 0; /* Sliced encoding not supported */
592         break;
593     case AV_CODEC_ID_H263:
594         if (!CONFIG_H263_ENCODER)
595         return -1;
596         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
597                              s->width, s->height) == 8) {
598             av_log(avctx, AV_LOG_INFO,
599                    "The specified picture size of %dx%d is not valid for "
600                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
601                    "352x288, 704x576, and 1408x1152."
602                    "Try H.263+.\n", s->width, s->height);
603             return -1;
604         }
605         s->out_format = FMT_H263;
606         avctx->delay  = 0;
607         s->low_delay  = 1;
608         break;
609     case AV_CODEC_ID_H263P:
610         s->out_format = FMT_H263;
611         s->h263_plus  = 1;
612         /* Fx */
613         s->h263_aic        = (avctx->flags & CODEC_FLAG_AC_PRED) ? 1 : 0;
614         s->modified_quant  = s->h263_aic;
615         s->loop_filter     = (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
616         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
617
618         /* /Fx */
619         /* These are just to be sure */
620         avctx->delay = 0;
621         s->low_delay = 1;
622         break;
623     case AV_CODEC_ID_FLV1:
624         s->out_format      = FMT_H263;
625         s->h263_flv        = 2; /* format = 1; 11-bit codes */
626         s->unrestricted_mv = 1;
627         s->rtp_mode  = 0; /* don't allow GOB */
628         avctx->delay = 0;
629         s->low_delay = 1;
630         break;
631     case AV_CODEC_ID_RV10:
632         s->out_format = FMT_H263;
633         avctx->delay  = 0;
634         s->low_delay  = 1;
635         break;
636     case AV_CODEC_ID_RV20:
637         s->out_format      = FMT_H263;
638         avctx->delay       = 0;
639         s->low_delay       = 1;
640         s->modified_quant  = 1;
641         s->h263_aic        = 1;
642         s->h263_plus       = 1;
643         s->loop_filter     = 1;
644         s->unrestricted_mv = 0;
645         break;
646     case AV_CODEC_ID_MPEG4:
647         s->out_format      = FMT_H263;
648         s->h263_pred       = 1;
649         s->unrestricted_mv = 1;
650         s->low_delay       = s->max_b_frames ? 0 : 1;
651         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
652         break;
653     case AV_CODEC_ID_MSMPEG4V2:
654         s->out_format      = FMT_H263;
655         s->h263_pred       = 1;
656         s->unrestricted_mv = 1;
657         s->msmpeg4_version = 2;
658         avctx->delay       = 0;
659         s->low_delay       = 1;
660         break;
661     case AV_CODEC_ID_MSMPEG4V3:
662         s->out_format        = FMT_H263;
663         s->h263_pred         = 1;
664         s->unrestricted_mv   = 1;
665         s->msmpeg4_version   = 3;
666         s->flipflop_rounding = 1;
667         avctx->delay         = 0;
668         s->low_delay         = 1;
669         break;
670     case AV_CODEC_ID_WMV1:
671         s->out_format        = FMT_H263;
672         s->h263_pred         = 1;
673         s->unrestricted_mv   = 1;
674         s->msmpeg4_version   = 4;
675         s->flipflop_rounding = 1;
676         avctx->delay         = 0;
677         s->low_delay         = 1;
678         break;
679     case AV_CODEC_ID_WMV2:
680         s->out_format        = FMT_H263;
681         s->h263_pred         = 1;
682         s->unrestricted_mv   = 1;
683         s->msmpeg4_version   = 5;
684         s->flipflop_rounding = 1;
685         avctx->delay         = 0;
686         s->low_delay         = 1;
687         break;
688     default:
689         return -1;
690     }
691
692     avctx->has_b_frames = !s->low_delay;
693
694     s->encoding = 1;
695
696     s->progressive_frame    =
697     s->progressive_sequence = !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT |
698                                                 CODEC_FLAG_INTERLACED_ME) ||
699                                 s->alternate_scan);
700
701     /* init */
702     ff_mpv_idct_init(s);
703     if (ff_mpv_common_init(s) < 0)
704         return -1;
705
706     if (ARCH_X86)
707         ff_mpv_encode_init_x86(s);
708
709     ff_fdctdsp_init(&s->fdsp, avctx);
710     ff_me_cmp_init(&s->mecc, avctx);
711     ff_mpegvideoencdsp_init(&s->mpvencdsp, avctx);
712     ff_pixblockdsp_init(&s->pdsp, avctx);
713     ff_qpeldsp_init(&s->qdsp);
714
715     s->avctx->coded_frame = s->current_picture.f;
716
717     if (s->msmpeg4_version) {
718         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
719                           2 * 2 * (MAX_LEVEL + 1) *
720                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
721     }
722     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
723
724     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
725     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
726     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
727     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
728     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
729                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
730     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
731                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
732
733     if (s->avctx->noise_reduction) {
734         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
735                           2 * 64 * sizeof(uint16_t), fail);
736     }
737
738     if (CONFIG_H263_ENCODER)
739         ff_h263dsp_init(&s->h263dsp);
740     if (!s->dct_quantize)
741         s->dct_quantize = ff_dct_quantize_c;
742     if (!s->denoise_dct)
743         s->denoise_dct  = denoise_dct_c;
744     s->fast_dct_quantize = s->dct_quantize;
745     if (avctx->trellis)
746         s->dct_quantize  = dct_quantize_trellis_c;
747
748     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
749         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
750
751     s->quant_precision = 5;
752
753     ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      s->avctx->ildct_cmp);
754     ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->avctx->frame_skip_cmp);
755
756     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
757         ff_h261_encode_init(s);
758     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
759         ff_h263_encode_init(s);
760     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
761         if ((ret = ff_msmpeg4_encode_init(s)) < 0)
762             return ret;
763     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
764         && s->out_format == FMT_MPEG1)
765         ff_mpeg1_encode_init(s);
766
767     /* init q matrix */
768     for (i = 0; i < 64; i++) {
769         int j = s->idsp.idct_permutation[i];
770         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
771             s->mpeg_quant) {
772             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
773             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
774         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
775             s->intra_matrix[j] =
776             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
777         } else {
778             /* mpeg1/2 */
779             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
780             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
781         }
782         if (s->avctx->intra_matrix)
783             s->intra_matrix[j] = s->avctx->intra_matrix[i];
784         if (s->avctx->inter_matrix)
785             s->inter_matrix[j] = s->avctx->inter_matrix[i];
786     }
787
788     /* precompute matrix */
789     /* for mjpeg, we do include qscale in the matrix */
790     if (s->out_format != FMT_MJPEG) {
791         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
792                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
793                           31, 1);
794         ff_convert_matrix(s, s->q_inter_matrix, s->q_inter_matrix16,
795                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
796                           31, 0);
797     }
798
799     if (ff_rate_control_init(s) < 0)
800         return -1;
801
802 #if FF_API_ERROR_RATE
803     FF_DISABLE_DEPRECATION_WARNINGS
804     if (avctx->error_rate)
805         s->error_rate = avctx->error_rate;
806     FF_ENABLE_DEPRECATION_WARNINGS;
807 #endif
808
809 #if FF_API_NORMALIZE_AQP
810     FF_DISABLE_DEPRECATION_WARNINGS
811     if (avctx->flags & CODEC_FLAG_NORMALIZE_AQP)
812         s->mpv_flags |= FF_MPV_FLAG_NAQ;
813     FF_ENABLE_DEPRECATION_WARNINGS;
814 #endif
815
816 #if FF_API_MV0
817     FF_DISABLE_DEPRECATION_WARNINGS
818     if (avctx->flags & CODEC_FLAG_MV0)
819         s->mpv_flags |= FF_MPV_FLAG_MV0;
820     FF_ENABLE_DEPRECATION_WARNINGS
821 #endif
822
823 #if FF_API_MPV_OPT
824     FF_DISABLE_DEPRECATION_WARNINGS
825     if (avctx->rc_qsquish != 0.0)
826         s->rc_qsquish = avctx->rc_qsquish;
827     if (avctx->rc_qmod_amp != 0.0)
828         s->rc_qmod_amp = avctx->rc_qmod_amp;
829     if (avctx->rc_qmod_freq)
830         s->rc_qmod_freq = avctx->rc_qmod_freq;
831     if (avctx->rc_buffer_aggressivity != 1.0)
832         s->rc_buffer_aggressivity = avctx->rc_buffer_aggressivity;
833     if (avctx->rc_initial_cplx != 0.0)
834         s->rc_initial_cplx = avctx->rc_initial_cplx;
835     if (avctx->lmin)
836         s->lmin = avctx->lmin;
837     if (avctx->lmax)
838         s->lmax = avctx->lmax;
839
840     if (avctx->rc_eq) {
841         av_freep(&s->rc_eq);
842         s->rc_eq = av_strdup(avctx->rc_eq);
843         if (!s->rc_eq)
844             return AVERROR(ENOMEM);
845     }
846     FF_ENABLE_DEPRECATION_WARNINGS
847 #endif
848
849     if (avctx->b_frame_strategy == 2) {
850         for (i = 0; i < s->max_b_frames + 2; i++) {
851             s->tmp_frames[i] = av_frame_alloc();
852             if (!s->tmp_frames[i])
853                 return AVERROR(ENOMEM);
854
855             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
856             s->tmp_frames[i]->width  = s->width  >> avctx->brd_scale;
857             s->tmp_frames[i]->height = s->height >> avctx->brd_scale;
858
859             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
860             if (ret < 0)
861                 return ret;
862         }
863     }
864
865     return 0;
866 fail:
867     ff_mpv_encode_end(avctx);
868     return AVERROR_UNKNOWN;
869 }
870
871 av_cold int ff_mpv_encode_end(AVCodecContext *avctx)
872 {
873     MpegEncContext *s = avctx->priv_data;
874     int i;
875
876     ff_rate_control_uninit(s);
877
878     ff_mpv_common_end(s);
879     if (CONFIG_MJPEG_ENCODER &&
880         s->out_format == FMT_MJPEG)
881         ff_mjpeg_encode_close(s);
882
883     av_freep(&avctx->extradata);
884
885     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
886         av_frame_free(&s->tmp_frames[i]);
887
888     ff_free_picture_tables(&s->new_picture);
889     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
890
891     av_freep(&s->avctx->stats_out);
892     av_freep(&s->ac_stats);
893
894     av_freep(&s->q_intra_matrix);
895     av_freep(&s->q_inter_matrix);
896     av_freep(&s->q_intra_matrix16);
897     av_freep(&s->q_inter_matrix16);
898     av_freep(&s->input_picture);
899     av_freep(&s->reordered_input_picture);
900     av_freep(&s->dct_offset);
901
902     return 0;
903 }
904
905 static int get_sae(uint8_t *src, int ref, int stride)
906 {
907     int x,y;
908     int acc = 0;
909
910     for (y = 0; y < 16; y++) {
911         for (x = 0; x < 16; x++) {
912             acc += FFABS(src[x + y * stride] - ref);
913         }
914     }
915
916     return acc;
917 }
918
919 static int get_intra_count(MpegEncContext *s, uint8_t *src,
920                            uint8_t *ref, int stride)
921 {
922     int x, y, w, h;
923     int acc = 0;
924
925     w = s->width  & ~15;
926     h = s->height & ~15;
927
928     for (y = 0; y < h; y += 16) {
929         for (x = 0; x < w; x += 16) {
930             int offset = x + y * stride;
931             int sad  = s->mecc.sad[0](NULL, src + offset, ref + offset,
932                                       stride, 16);
933             int mean = (s->mpvencdsp.pix_sum(src + offset, stride) + 128) >> 8;
934             int sae  = get_sae(src + offset, mean, stride);
935
936             acc += sae + 500 < sad;
937         }
938     }
939     return acc;
940 }
941
942
943 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
944 {
945     Picture *pic = NULL;
946     int64_t pts;
947     int i, display_picture_number = 0, ret;
948     const int encoding_delay = s->max_b_frames ? s->max_b_frames :
949                                                  (s->low_delay ? 0 : 1);
950     int direct = 1;
951
952     if (pic_arg) {
953         pts = pic_arg->pts;
954         display_picture_number = s->input_picture_number++;
955
956         if (pts != AV_NOPTS_VALUE) {
957             if (s->user_specified_pts != AV_NOPTS_VALUE) {
958                 int64_t time = pts;
959                 int64_t last = s->user_specified_pts;
960
961                 if (time <= last) {
962                     av_log(s->avctx, AV_LOG_ERROR,
963                            "Error, Invalid timestamp=%"PRId64", "
964                            "last=%"PRId64"\n", pts, s->user_specified_pts);
965                     return -1;
966                 }
967
968                 if (!s->low_delay && display_picture_number == 1)
969                     s->dts_delta = time - last;
970             }
971             s->user_specified_pts = pts;
972         } else {
973             if (s->user_specified_pts != AV_NOPTS_VALUE) {
974                 s->user_specified_pts =
975                 pts = s->user_specified_pts + 1;
976                 av_log(s->avctx, AV_LOG_INFO,
977                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
978                        pts);
979             } else {
980                 pts = display_picture_number;
981             }
982         }
983     }
984
985     if (pic_arg) {
986         if (!pic_arg->buf[0] ||
987             pic_arg->linesize[0] != s->linesize ||
988             pic_arg->linesize[1] != s->uvlinesize ||
989             pic_arg->linesize[2] != s->uvlinesize)
990             direct = 0;
991         if ((s->width & 15) || (s->height & 15))
992             direct = 0;
993
994         ff_dlog(s->avctx, "%d %d %td %td\n", pic_arg->linesize[0],
995                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
996
997         i = ff_find_unused_picture(s->avctx, s->picture, direct);
998         if (i < 0)
999             return i;
1000
1001         pic = &s->picture[i];
1002         pic->reference = 3;
1003
1004         if (direct) {
1005             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
1006                 return ret;
1007         }
1008         ret = ff_alloc_picture(s, pic, direct);
1009         if (ret < 0)
1010             return ret;
1011
1012         if (!direct) {
1013             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1014                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1015                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1016                 // empty
1017             } else {
1018                 int h_chroma_shift, v_chroma_shift;
1019                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
1020                                                  &h_chroma_shift,
1021                                                  &v_chroma_shift);
1022
1023                 for (i = 0; i < 3; i++) {
1024                     int src_stride = pic_arg->linesize[i];
1025                     int dst_stride = i ? s->uvlinesize : s->linesize;
1026                     int h_shift = i ? h_chroma_shift : 0;
1027                     int v_shift = i ? v_chroma_shift : 0;
1028                     int w = s->width  >> h_shift;
1029                     int h = s->height >> v_shift;
1030                     uint8_t *src = pic_arg->data[i];
1031                     uint8_t *dst = pic->f->data[i];
1032
1033                     if (!s->avctx->rc_buffer_size)
1034                         dst += INPLACE_OFFSET;
1035
1036                     if (src_stride == dst_stride)
1037                         memcpy(dst, src, src_stride * h);
1038                     else {
1039                         int h2 = h;
1040                         uint8_t *dst2 = dst;
1041                         while (h2--) {
1042                             memcpy(dst2, src, w);
1043                             dst2 += dst_stride;
1044                             src += src_stride;
1045                         }
1046                     }
1047                     if ((s->width & 15) || (s->height & 15)) {
1048                         s->mpvencdsp.draw_edges(dst, dst_stride,
1049                                                 w, h,
1050                                                 16 >> h_shift,
1051                                                 16 >> v_shift,
1052                                                 EDGE_BOTTOM);
1053                     }
1054                 }
1055             }
1056         }
1057         ret = av_frame_copy_props(pic->f, pic_arg);
1058         if (ret < 0)
1059             return ret;
1060
1061         pic->f->display_picture_number = display_picture_number;
1062         pic->f->pts = pts; // we set this here to avoid modifiying pic_arg
1063     }
1064
1065     /* shift buffer entries */
1066     for (i = 1; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1067         s->input_picture[i - 1] = s->input_picture[i];
1068
1069     s->input_picture[encoding_delay] = (Picture*) pic;
1070
1071     return 0;
1072 }
1073
1074 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1075 {
1076     int x, y, plane;
1077     int score = 0;
1078     int64_t score64 = 0;
1079
1080     for (plane = 0; plane < 3; plane++) {
1081         const int stride = p->f->linesize[plane];
1082         const int bw = plane ? 1 : 2;
1083         for (y = 0; y < s->mb_height * bw; y++) {
1084             for (x = 0; x < s->mb_width * bw; x++) {
1085                 int off = p->shared ? 0 : 16;
1086                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1087                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1088                 int v = s->mecc.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1089
1090                 switch (s->avctx->frame_skip_exp) {
1091                 case 0: score    =  FFMAX(score, v);          break;
1092                 case 1: score   += FFABS(v);                  break;
1093                 case 2: score   += v * v;                     break;
1094                 case 3: score64 += FFABS(v * v * (int64_t)v); break;
1095                 case 4: score64 += v * v * (int64_t)(v * v);  break;
1096                 }
1097             }
1098         }
1099     }
1100
1101     if (score)
1102         score64 = score;
1103
1104     if (score64 < s->avctx->frame_skip_threshold)
1105         return 1;
1106     if (score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda) >> 8))
1107         return 1;
1108     return 0;
1109 }
1110
1111 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1112 {
1113     AVPacket pkt = { 0 };
1114     int ret, got_output;
1115
1116     av_init_packet(&pkt);
1117     ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
1118     if (ret < 0)
1119         return ret;
1120
1121     ret = pkt.size;
1122     av_free_packet(&pkt);
1123     return ret;
1124 }
1125
1126 static int estimate_best_b_count(MpegEncContext *s)
1127 {
1128     AVCodec *codec    = avcodec_find_encoder(s->avctx->codec_id);
1129     AVCodecContext *c = avcodec_alloc_context3(NULL);
1130     const int scale = s->avctx->brd_scale;
1131     int i, j, out_size, p_lambda, b_lambda, lambda2;
1132     int64_t best_rd  = INT64_MAX;
1133     int best_b_count = -1;
1134
1135     if (!c)
1136         return AVERROR(ENOMEM);
1137     assert(scale >= 0 && scale <= 3);
1138
1139     //emms_c();
1140     //s->next_picture_ptr->quality;
1141     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1142     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1143     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1144     if (!b_lambda) // FIXME we should do this somewhere else
1145         b_lambda = p_lambda;
1146     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1147                FF_LAMBDA_SHIFT;
1148
1149     c->width        = s->width  >> scale;
1150     c->height       = s->height >> scale;
1151     c->flags        = CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR;
1152     c->flags       |= s->avctx->flags & CODEC_FLAG_QPEL;
1153     c->mb_decision  = s->avctx->mb_decision;
1154     c->me_cmp       = s->avctx->me_cmp;
1155     c->mb_cmp       = s->avctx->mb_cmp;
1156     c->me_sub_cmp   = s->avctx->me_sub_cmp;
1157     c->pix_fmt      = AV_PIX_FMT_YUV420P;
1158     c->time_base    = s->avctx->time_base;
1159     c->max_b_frames = s->max_b_frames;
1160
1161     if (avcodec_open2(c, codec, NULL) < 0)
1162         return -1;
1163
1164     for (i = 0; i < s->max_b_frames + 2; i++) {
1165         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1166                                                 s->next_picture_ptr;
1167
1168         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1169             pre_input = *pre_input_ptr;
1170
1171             if (!pre_input.shared && i) {
1172                 pre_input.f->data[0] += INPLACE_OFFSET;
1173                 pre_input.f->data[1] += INPLACE_OFFSET;
1174                 pre_input.f->data[2] += INPLACE_OFFSET;
1175             }
1176
1177             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[0],
1178                                        s->tmp_frames[i]->linesize[0],
1179                                        pre_input.f->data[0],
1180                                        pre_input.f->linesize[0],
1181                                        c->width, c->height);
1182             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[1],
1183                                        s->tmp_frames[i]->linesize[1],
1184                                        pre_input.f->data[1],
1185                                        pre_input.f->linesize[1],
1186                                        c->width >> 1, c->height >> 1);
1187             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[2],
1188                                        s->tmp_frames[i]->linesize[2],
1189                                        pre_input.f->data[2],
1190                                        pre_input.f->linesize[2],
1191                                        c->width >> 1, c->height >> 1);
1192         }
1193     }
1194
1195     for (j = 0; j < s->max_b_frames + 1; j++) {
1196         int64_t rd = 0;
1197
1198         if (!s->input_picture[j])
1199             break;
1200
1201         c->error[0] = c->error[1] = c->error[2] = 0;
1202
1203         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1204         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1205
1206         out_size = encode_frame(c, s->tmp_frames[0]);
1207
1208         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1209
1210         for (i = 0; i < s->max_b_frames + 1; i++) {
1211             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1212
1213             s->tmp_frames[i + 1]->pict_type = is_p ?
1214                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1215             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1216
1217             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1218
1219             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1220         }
1221
1222         /* get the delayed frames */
1223         while (out_size) {
1224             out_size = encode_frame(c, NULL);
1225             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1226         }
1227
1228         rd += c->error[0] + c->error[1] + c->error[2];
1229
1230         if (rd < best_rd) {
1231             best_rd = rd;
1232             best_b_count = j;
1233         }
1234     }
1235
1236     avcodec_close(c);
1237     av_freep(&c);
1238
1239     return best_b_count;
1240 }
1241
1242 static int select_input_picture(MpegEncContext *s)
1243 {
1244     int i, ret;
1245
1246     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1247         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1248     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1249
1250     /* set next picture type & ordering */
1251     if (!s->reordered_input_picture[0] && s->input_picture[0]) {
1252         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1253             !s->next_picture_ptr || s->intra_only) {
1254             s->reordered_input_picture[0] = s->input_picture[0];
1255             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1256             s->reordered_input_picture[0]->f->coded_picture_number =
1257                 s->coded_picture_number++;
1258         } else {
1259             int b_frames;
1260
1261             if (s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor) {
1262                 if (s->picture_in_gop_number < s->gop_size &&
1263                     skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1264                     // FIXME check that te gop check above is +-1 correct
1265                     av_frame_unref(s->input_picture[0]->f);
1266
1267                     emms_c();
1268                     ff_vbv_update(s, 0);
1269
1270                     goto no_output_pic;
1271                 }
1272             }
1273
1274             if (s->avctx->flags & CODEC_FLAG_PASS2) {
1275                 for (i = 0; i < s->max_b_frames + 1; i++) {
1276                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1277
1278                     if (pict_num >= s->rc_context.num_entries)
1279                         break;
1280                     if (!s->input_picture[i]) {
1281                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1282                         break;
1283                     }
1284
1285                     s->input_picture[i]->f->pict_type =
1286                         s->rc_context.entry[pict_num].new_pict_type;
1287                 }
1288             }
1289
1290             if (s->avctx->b_frame_strategy == 0) {
1291                 b_frames = s->max_b_frames;
1292                 while (b_frames && !s->input_picture[b_frames])
1293                     b_frames--;
1294             } else if (s->avctx->b_frame_strategy == 1) {
1295                 for (i = 1; i < s->max_b_frames + 1; i++) {
1296                     if (s->input_picture[i] &&
1297                         s->input_picture[i]->b_frame_score == 0) {
1298                         s->input_picture[i]->b_frame_score =
1299                             get_intra_count(s,
1300                                             s->input_picture[i    ]->f->data[0],
1301                                             s->input_picture[i - 1]->f->data[0],
1302                                             s->linesize) + 1;
1303                     }
1304                 }
1305                 for (i = 0; i < s->max_b_frames + 1; i++) {
1306                     if (!s->input_picture[i] ||
1307                         s->input_picture[i]->b_frame_score - 1 >
1308                             s->mb_num / s->avctx->b_sensitivity)
1309                         break;
1310                 }
1311
1312                 b_frames = FFMAX(0, i - 1);
1313
1314                 /* reset scores */
1315                 for (i = 0; i < b_frames + 1; i++) {
1316                     s->input_picture[i]->b_frame_score = 0;
1317                 }
1318             } else if (s->avctx->b_frame_strategy == 2) {
1319                 b_frames = estimate_best_b_count(s);
1320             } else {
1321                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1322                 b_frames = 0;
1323             }
1324
1325             emms_c();
1326
1327             for (i = b_frames - 1; i >= 0; i--) {
1328                 int type = s->input_picture[i]->f->pict_type;
1329                 if (type && type != AV_PICTURE_TYPE_B)
1330                     b_frames = i;
1331             }
1332             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1333                 b_frames == s->max_b_frames) {
1334                 av_log(s->avctx, AV_LOG_ERROR,
1335                        "warning, too many b frames in a row\n");
1336             }
1337
1338             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1339                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1340                     s->gop_size > s->picture_in_gop_number) {
1341                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1342                 } else {
1343                     if (s->avctx->flags & CODEC_FLAG_CLOSED_GOP)
1344                         b_frames = 0;
1345                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1346                 }
1347             }
1348
1349             if ((s->avctx->flags & CODEC_FLAG_CLOSED_GOP) && b_frames &&
1350                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1351                 b_frames--;
1352
1353             s->reordered_input_picture[0] = s->input_picture[b_frames];
1354             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1355                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1356             s->reordered_input_picture[0]->f->coded_picture_number =
1357                 s->coded_picture_number++;
1358             for (i = 0; i < b_frames; i++) {
1359                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1360                 s->reordered_input_picture[i + 1]->f->pict_type =
1361                     AV_PICTURE_TYPE_B;
1362                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1363                     s->coded_picture_number++;
1364             }
1365         }
1366     }
1367 no_output_pic:
1368     if (s->reordered_input_picture[0]) {
1369         s->reordered_input_picture[0]->reference =
1370            s->reordered_input_picture[0]->f->pict_type !=
1371                AV_PICTURE_TYPE_B ? 3 : 0;
1372
1373         ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1374         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->new_picture, s->reordered_input_picture[0])))
1375             return ret;
1376
1377         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1378             // input is a shared pix, so we can't modifiy it -> alloc a new
1379             // one & ensure that the shared one is reuseable
1380
1381             Picture *pic;
1382             int i = ff_find_unused_picture(s->avctx, s->picture, 0);
1383             if (i < 0)
1384                 return i;
1385             pic = &s->picture[i];
1386
1387             pic->reference = s->reordered_input_picture[0]->reference;
1388             if (ff_alloc_picture(s, pic, 0) < 0) {
1389                 return -1;
1390             }
1391
1392             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1393             if (ret < 0)
1394                 return ret;
1395
1396             /* mark us unused / free shared pic */
1397             av_frame_unref(s->reordered_input_picture[0]->f);
1398             s->reordered_input_picture[0]->shared = 0;
1399
1400             s->current_picture_ptr = pic;
1401         } else {
1402             // input is not a shared pix -> reuse buffer for current_pix
1403             s->current_picture_ptr = s->reordered_input_picture[0];
1404             for (i = 0; i < 4; i++) {
1405                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1406             }
1407         }
1408         ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1409         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1410                                        s->current_picture_ptr)) < 0)
1411             return ret;
1412
1413         s->picture_number = s->new_picture.f->display_picture_number;
1414     } else {
1415         ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1416     }
1417     return 0;
1418 }
1419
1420 static void frame_end(MpegEncContext *s)
1421 {
1422     int i;
1423
1424     if (s->unrestricted_mv &&
1425         s->current_picture.reference &&
1426         !s->intra_only) {
1427         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1428         int hshift = desc->log2_chroma_w;
1429         int vshift = desc->log2_chroma_h;
1430         s->mpvencdsp.draw_edges(s->current_picture.f->data[0], s->linesize,
1431                                 s->h_edge_pos, s->v_edge_pos,
1432                                 EDGE_WIDTH, EDGE_WIDTH,
1433                                 EDGE_TOP | EDGE_BOTTOM);
1434         s->mpvencdsp.draw_edges(s->current_picture.f->data[1], s->uvlinesize,
1435                                 s->h_edge_pos >> hshift,
1436                                 s->v_edge_pos >> vshift,
1437                                 EDGE_WIDTH >> hshift,
1438                                 EDGE_WIDTH >> vshift,
1439                                 EDGE_TOP | EDGE_BOTTOM);
1440         s->mpvencdsp.draw_edges(s->current_picture.f->data[2], s->uvlinesize,
1441                                 s->h_edge_pos >> hshift,
1442                                 s->v_edge_pos >> vshift,
1443                                 EDGE_WIDTH >> hshift,
1444                                 EDGE_WIDTH >> vshift,
1445                                 EDGE_TOP | EDGE_BOTTOM);
1446     }
1447
1448     emms_c();
1449
1450     s->last_pict_type                 = s->pict_type;
1451     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1452     if (s->pict_type!= AV_PICTURE_TYPE_B)
1453         s->last_non_b_pict_type = s->pict_type;
1454
1455     if (s->encoding) {
1456         /* release non-reference frames */
1457         for (i = 0; i < MAX_PICTURE_COUNT; i++) {
1458             if (!s->picture[i].reference)
1459                 ff_mpeg_unref_picture(s->avctx, &s->picture[i]);
1460         }
1461     }
1462
1463     s->avctx->coded_frame = s->current_picture_ptr->f;
1464
1465 }
1466
1467 static void update_noise_reduction(MpegEncContext *s)
1468 {
1469     int intra, i;
1470
1471     for (intra = 0; intra < 2; intra++) {
1472         if (s->dct_count[intra] > (1 << 16)) {
1473             for (i = 0; i < 64; i++) {
1474                 s->dct_error_sum[intra][i] >>= 1;
1475             }
1476             s->dct_count[intra] >>= 1;
1477         }
1478
1479         for (i = 0; i < 64; i++) {
1480             s->dct_offset[intra][i] = (s->avctx->noise_reduction *
1481                                        s->dct_count[intra] +
1482                                        s->dct_error_sum[intra][i] / 2) /
1483                                       (s->dct_error_sum[intra][i] + 1);
1484         }
1485     }
1486 }
1487
1488 static int frame_start(MpegEncContext *s)
1489 {
1490     int ret;
1491
1492     /* mark & release old frames */
1493     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1494         s->last_picture_ptr != s->next_picture_ptr &&
1495         s->last_picture_ptr->f->buf[0]) {
1496         ff_mpeg_unref_picture(s->avctx, s->last_picture_ptr);
1497     }
1498
1499     s->current_picture_ptr->f->pict_type = s->pict_type;
1500     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1501
1502     ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1503     if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1504                                    s->current_picture_ptr)) < 0)
1505         return ret;
1506
1507     if (s->pict_type != AV_PICTURE_TYPE_B) {
1508         s->last_picture_ptr = s->next_picture_ptr;
1509         if (!s->droppable)
1510             s->next_picture_ptr = s->current_picture_ptr;
1511     }
1512
1513     if (s->last_picture_ptr) {
1514         ff_mpeg_unref_picture(s->avctx, &s->last_picture);
1515         if (s->last_picture_ptr->f->buf[0] &&
1516             (ret = ff_mpeg_ref_picture(s->avctx, &s->last_picture,
1517                                        s->last_picture_ptr)) < 0)
1518             return ret;
1519     }
1520     if (s->next_picture_ptr) {
1521         ff_mpeg_unref_picture(s->avctx, &s->next_picture);
1522         if (s->next_picture_ptr->f->buf[0] &&
1523             (ret = ff_mpeg_ref_picture(s->avctx, &s->next_picture,
1524                                        s->next_picture_ptr)) < 0)
1525             return ret;
1526     }
1527
1528     if (s->picture_structure!= PICT_FRAME) {
1529         int i;
1530         for (i = 0; i < 4; i++) {
1531             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1532                 s->current_picture.f->data[i] +=
1533                     s->current_picture.f->linesize[i];
1534             }
1535             s->current_picture.f->linesize[i] *= 2;
1536             s->last_picture.f->linesize[i]    *= 2;
1537             s->next_picture.f->linesize[i]    *= 2;
1538         }
1539     }
1540
1541     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1542         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1543         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1544     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1545         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1546         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1547     } else {
1548         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1549         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1550     }
1551
1552     if (s->dct_error_sum) {
1553         assert(s->avctx->noise_reduction && s->encoding);
1554         update_noise_reduction(s);
1555     }
1556
1557     return 0;
1558 }
1559
1560 int ff_mpv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1561                           const AVFrame *pic_arg, int *got_packet)
1562 {
1563     MpegEncContext *s = avctx->priv_data;
1564     int i, stuffing_count, ret;
1565     int context_count = s->slice_context_count;
1566
1567     s->picture_in_gop_number++;
1568
1569     if (load_input_picture(s, pic_arg) < 0)
1570         return -1;
1571
1572     if (select_input_picture(s) < 0) {
1573         return -1;
1574     }
1575
1576     /* output? */
1577     if (s->new_picture.f->data[0]) {
1578         if (!pkt->data &&
1579             (ret = ff_alloc_packet(pkt, s->mb_width*s->mb_height*MAX_MB_BYTES)) < 0)
1580             return ret;
1581         if (s->mb_info) {
1582             s->mb_info_ptr = av_packet_new_side_data(pkt,
1583                                  AV_PKT_DATA_H263_MB_INFO,
1584                                  s->mb_width*s->mb_height*12);
1585             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1586         }
1587
1588         for (i = 0; i < context_count; i++) {
1589             int start_y = s->thread_context[i]->start_mb_y;
1590             int   end_y = s->thread_context[i]->  end_mb_y;
1591             int h       = s->mb_height;
1592             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1593             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1594
1595             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1596         }
1597
1598         s->pict_type = s->new_picture.f->pict_type;
1599         //emms_c();
1600         ret = frame_start(s);
1601         if (ret < 0)
1602             return ret;
1603 vbv_retry:
1604         if (encode_picture(s, s->picture_number) < 0)
1605             return -1;
1606
1607         avctx->header_bits = s->header_bits;
1608         avctx->mv_bits     = s->mv_bits;
1609         avctx->misc_bits   = s->misc_bits;
1610         avctx->i_tex_bits  = s->i_tex_bits;
1611         avctx->p_tex_bits  = s->p_tex_bits;
1612         avctx->i_count     = s->i_count;
1613         // FIXME f/b_count in avctx
1614         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1615         avctx->skip_count  = s->skip_count;
1616
1617         frame_end(s);
1618
1619         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1620             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1621
1622         if (avctx->rc_buffer_size) {
1623             RateControlContext *rcc = &s->rc_context;
1624             int max_size = rcc->buffer_index * avctx->rc_max_available_vbv_use;
1625
1626             if (put_bits_count(&s->pb) > max_size &&
1627                 s->lambda < s->lmax) {
1628                 s->next_lambda = FFMAX(s->lambda + 1, s->lambda *
1629                                        (s->qscale + 1) / s->qscale);
1630                 if (s->adaptive_quant) {
1631                     int i;
1632                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1633                         s->lambda_table[i] =
1634                             FFMAX(s->lambda_table[i] + 1,
1635                                   s->lambda_table[i] * (s->qscale + 1) /
1636                                   s->qscale);
1637                 }
1638                 s->mb_skipped = 0;        // done in frame_start()
1639                 // done in encode_picture() so we must undo it
1640                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1641                     if (s->flipflop_rounding          ||
1642                         s->codec_id == AV_CODEC_ID_H263P ||
1643                         s->codec_id == AV_CODEC_ID_MPEG4)
1644                         s->no_rounding ^= 1;
1645                 }
1646                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1647                     s->time_base       = s->last_time_base;
1648                     s->last_non_b_time = s->time - s->pp_time;
1649                 }
1650                 for (i = 0; i < context_count; i++) {
1651                     PutBitContext *pb = &s->thread_context[i]->pb;
1652                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1653                 }
1654                 goto vbv_retry;
1655             }
1656
1657             assert(s->avctx->rc_max_rate);
1658         }
1659
1660         if (s->avctx->flags & CODEC_FLAG_PASS1)
1661             ff_write_pass1_stats(s);
1662
1663         for (i = 0; i < 4; i++) {
1664             s->current_picture_ptr->f->error[i] = s->current_picture.f->error[i];
1665             avctx->error[i] += s->current_picture_ptr->f->error[i];
1666         }
1667
1668         if (s->avctx->flags & CODEC_FLAG_PASS1)
1669             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits +
1670                    avctx->i_tex_bits + avctx->p_tex_bits ==
1671                        put_bits_count(&s->pb));
1672         flush_put_bits(&s->pb);
1673         s->frame_bits  = put_bits_count(&s->pb);
1674
1675         stuffing_count = ff_vbv_update(s, s->frame_bits);
1676         if (stuffing_count) {
1677             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1678                     stuffing_count + 50) {
1679                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1680                 return -1;
1681             }
1682
1683             switch (s->codec_id) {
1684             case AV_CODEC_ID_MPEG1VIDEO:
1685             case AV_CODEC_ID_MPEG2VIDEO:
1686                 while (stuffing_count--) {
1687                     put_bits(&s->pb, 8, 0);
1688                 }
1689             break;
1690             case AV_CODEC_ID_MPEG4:
1691                 put_bits(&s->pb, 16, 0);
1692                 put_bits(&s->pb, 16, 0x1C3);
1693                 stuffing_count -= 4;
1694                 while (stuffing_count--) {
1695                     put_bits(&s->pb, 8, 0xFF);
1696                 }
1697             break;
1698             default:
1699                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1700             }
1701             flush_put_bits(&s->pb);
1702             s->frame_bits  = put_bits_count(&s->pb);
1703         }
1704
1705         /* update mpeg1/2 vbv_delay for CBR */
1706         if (s->avctx->rc_max_rate                          &&
1707             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1708             s->out_format == FMT_MPEG1                     &&
1709             90000LL * (avctx->rc_buffer_size - 1) <=
1710                 s->avctx->rc_max_rate * 0xFFFFLL) {
1711             int vbv_delay, min_delay;
1712             double inbits  = s->avctx->rc_max_rate *
1713                              av_q2d(s->avctx->time_base);
1714             int    minbits = s->frame_bits - 8 *
1715                              (s->vbv_delay_ptr - s->pb.buf - 1);
1716             double bits    = s->rc_context.buffer_index + minbits - inbits;
1717
1718             if (bits < 0)
1719                 av_log(s->avctx, AV_LOG_ERROR,
1720                        "Internal error, negative bits\n");
1721
1722             assert(s->repeat_first_field == 0);
1723
1724             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
1725             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
1726                         s->avctx->rc_max_rate;
1727
1728             vbv_delay = FFMAX(vbv_delay, min_delay);
1729
1730             assert(vbv_delay < 0xFFFF);
1731
1732             s->vbv_delay_ptr[0] &= 0xF8;
1733             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
1734             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
1735             s->vbv_delay_ptr[2] &= 0x07;
1736             s->vbv_delay_ptr[2] |= vbv_delay << 3;
1737             avctx->vbv_delay     = vbv_delay * 300;
1738         }
1739         s->total_bits     += s->frame_bits;
1740         avctx->frame_bits  = s->frame_bits;
1741
1742         pkt->pts = s->current_picture.f->pts;
1743         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
1744             if (!s->current_picture.f->coded_picture_number)
1745                 pkt->dts = pkt->pts - s->dts_delta;
1746             else
1747                 pkt->dts = s->reordered_pts;
1748             s->reordered_pts = pkt->pts;
1749         } else
1750             pkt->dts = pkt->pts;
1751         if (s->current_picture.f->key_frame)
1752             pkt->flags |= AV_PKT_FLAG_KEY;
1753         if (s->mb_info)
1754             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
1755     } else {
1756         s->frame_bits = 0;
1757     }
1758     assert((s->frame_bits & 7) == 0);
1759
1760     pkt->size = s->frame_bits / 8;
1761     *got_packet = !!pkt->size;
1762     return 0;
1763 }
1764
1765 static inline void dct_single_coeff_elimination(MpegEncContext *s,
1766                                                 int n, int threshold)
1767 {
1768     static const char tab[64] = {
1769         3, 2, 2, 1, 1, 1, 1, 1,
1770         1, 1, 1, 1, 1, 1, 1, 1,
1771         1, 1, 1, 1, 1, 1, 1, 1,
1772         0, 0, 0, 0, 0, 0, 0, 0,
1773         0, 0, 0, 0, 0, 0, 0, 0,
1774         0, 0, 0, 0, 0, 0, 0, 0,
1775         0, 0, 0, 0, 0, 0, 0, 0,
1776         0, 0, 0, 0, 0, 0, 0, 0
1777     };
1778     int score = 0;
1779     int run = 0;
1780     int i;
1781     int16_t *block = s->block[n];
1782     const int last_index = s->block_last_index[n];
1783     int skip_dc;
1784
1785     if (threshold < 0) {
1786         skip_dc = 0;
1787         threshold = -threshold;
1788     } else
1789         skip_dc = 1;
1790
1791     /* Are all we could set to zero already zero? */
1792     if (last_index <= skip_dc - 1)
1793         return;
1794
1795     for (i = 0; i <= last_index; i++) {
1796         const int j = s->intra_scantable.permutated[i];
1797         const int level = FFABS(block[j]);
1798         if (level == 1) {
1799             if (skip_dc && i == 0)
1800                 continue;
1801             score += tab[run];
1802             run = 0;
1803         } else if (level > 1) {
1804             return;
1805         } else {
1806             run++;
1807         }
1808     }
1809     if (score >= threshold)
1810         return;
1811     for (i = skip_dc; i <= last_index; i++) {
1812         const int j = s->intra_scantable.permutated[i];
1813         block[j] = 0;
1814     }
1815     if (block[0])
1816         s->block_last_index[n] = 0;
1817     else
1818         s->block_last_index[n] = -1;
1819 }
1820
1821 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
1822                                int last_index)
1823 {
1824     int i;
1825     const int maxlevel = s->max_qcoeff;
1826     const int minlevel = s->min_qcoeff;
1827     int overflow = 0;
1828
1829     if (s->mb_intra) {
1830         i = 1; // skip clipping of intra dc
1831     } else
1832         i = 0;
1833
1834     for (; i <= last_index; i++) {
1835         const int j = s->intra_scantable.permutated[i];
1836         int level = block[j];
1837
1838         if (level > maxlevel) {
1839             level = maxlevel;
1840             overflow++;
1841         } else if (level < minlevel) {
1842             level = minlevel;
1843             overflow++;
1844         }
1845
1846         block[j] = level;
1847     }
1848
1849     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1850         av_log(s->avctx, AV_LOG_INFO,
1851                "warning, clipping %d dct coefficients to %d..%d\n",
1852                overflow, minlevel, maxlevel);
1853 }
1854
1855 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
1856 {
1857     int x, y;
1858     // FIXME optimize
1859     for (y = 0; y < 8; y++) {
1860         for (x = 0; x < 8; x++) {
1861             int x2, y2;
1862             int sum = 0;
1863             int sqr = 0;
1864             int count = 0;
1865
1866             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
1867                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
1868                     int v = ptr[x2 + y2 * stride];
1869                     sum += v;
1870                     sqr += v * v;
1871                     count++;
1872                 }
1873             }
1874             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
1875         }
1876     }
1877 }
1878
1879 static av_always_inline void encode_mb_internal(MpegEncContext *s,
1880                                                 int motion_x, int motion_y,
1881                                                 int mb_block_height,
1882                                                 int mb_block_count)
1883 {
1884     int16_t weight[8][64];
1885     int16_t orig[8][64];
1886     const int mb_x = s->mb_x;
1887     const int mb_y = s->mb_y;
1888     int i;
1889     int skip_dct[8];
1890     int dct_offset = s->linesize * 8; // default for progressive frames
1891     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1892     ptrdiff_t wrap_y, wrap_c;
1893
1894     for (i = 0; i < mb_block_count; i++)
1895         skip_dct[i] = s->skipdct;
1896
1897     if (s->adaptive_quant) {
1898         const int last_qp = s->qscale;
1899         const int mb_xy = mb_x + mb_y * s->mb_stride;
1900
1901         s->lambda = s->lambda_table[mb_xy];
1902         update_qscale(s);
1903
1904         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
1905             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
1906             s->dquant = s->qscale - last_qp;
1907
1908             if (s->out_format == FMT_H263) {
1909                 s->dquant = av_clip(s->dquant, -2, 2);
1910
1911                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
1912                     if (!s->mb_intra) {
1913                         if (s->pict_type == AV_PICTURE_TYPE_B) {
1914                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
1915                                 s->dquant = 0;
1916                         }
1917                         if (s->mv_type == MV_TYPE_8X8)
1918                             s->dquant = 0;
1919                     }
1920                 }
1921             }
1922         }
1923         ff_set_qscale(s, last_qp + s->dquant);
1924     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
1925         ff_set_qscale(s, s->qscale + s->dquant);
1926
1927     wrap_y = s->linesize;
1928     wrap_c = s->uvlinesize;
1929     ptr_y  = s->new_picture.f->data[0] +
1930              (mb_y * 16 * wrap_y)              + mb_x * 16;
1931     ptr_cb = s->new_picture.f->data[1] +
1932              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1933     ptr_cr = s->new_picture.f->data[2] +
1934              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1935
1936     if (mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) {
1937         uint8_t *ebuf = s->edge_emu_buffer + 32;
1938         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
1939                                  wrap_y, wrap_y,
1940                                  16, 16, mb_x * 16, mb_y * 16,
1941                                  s->width, s->height);
1942         ptr_y = ebuf;
1943         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y, ptr_cb,
1944                                  wrap_c, wrap_c,
1945                                  8, mb_block_height, mb_x * 8, mb_y * 8,
1946                                  s->width >> 1, s->height >> 1);
1947         ptr_cb = ebuf + 18 * wrap_y;
1948         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y + 8, ptr_cr,
1949                                  wrap_c, wrap_c,
1950                                  8, mb_block_height, mb_x * 8, mb_y * 8,
1951                                  s->width >> 1, s->height >> 1);
1952         ptr_cr = ebuf + 18 * wrap_y + 8;
1953     }
1954
1955     if (s->mb_intra) {
1956         if (s->avctx->flags & CODEC_FLAG_INTERLACED_DCT) {
1957             int progressive_score, interlaced_score;
1958
1959             s->interlaced_dct = 0;
1960             progressive_score = s->mecc.ildct_cmp[4](s, ptr_y, NULL, wrap_y, 8) +
1961                                 s->mecc.ildct_cmp[4](s, ptr_y + wrap_y * 8,
1962                                                      NULL, wrap_y, 8) - 400;
1963
1964             if (progressive_score > 0) {
1965                 interlaced_score = s->mecc.ildct_cmp[4](s, ptr_y,
1966                                                         NULL, wrap_y * 2, 8) +
1967                                    s->mecc.ildct_cmp[4](s, ptr_y + wrap_y,
1968                                                         NULL, wrap_y * 2, 8);
1969                 if (progressive_score > interlaced_score) {
1970                     s->interlaced_dct = 1;
1971
1972                     dct_offset = wrap_y;
1973                     wrap_y <<= 1;
1974                     if (s->chroma_format == CHROMA_422)
1975                         wrap_c <<= 1;
1976                 }
1977             }
1978         }
1979
1980         s->pdsp.get_pixels(s->block[0], ptr_y,                  wrap_y);
1981         s->pdsp.get_pixels(s->block[1], ptr_y + 8,              wrap_y);
1982         s->pdsp.get_pixels(s->block[2], ptr_y + dct_offset,     wrap_y);
1983         s->pdsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
1984
1985         if (s->avctx->flags & CODEC_FLAG_GRAY) {
1986             skip_dct[4] = 1;
1987             skip_dct[5] = 1;
1988         } else {
1989             s->pdsp.get_pixels(s->block[4], ptr_cb, wrap_c);
1990             s->pdsp.get_pixels(s->block[5], ptr_cr, wrap_c);
1991             if (!s->chroma_y_shift) { /* 422 */
1992                 s->pdsp.get_pixels(s->block[6],
1993                                    ptr_cb + (dct_offset >> 1), wrap_c);
1994                 s->pdsp.get_pixels(s->block[7],
1995                                    ptr_cr + (dct_offset >> 1), wrap_c);
1996             }
1997         }
1998     } else {
1999         op_pixels_func (*op_pix)[4];
2000         qpel_mc_func (*op_qpix)[16];
2001         uint8_t *dest_y, *dest_cb, *dest_cr;
2002
2003         dest_y  = s->dest[0];
2004         dest_cb = s->dest[1];
2005         dest_cr = s->dest[2];
2006
2007         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
2008             op_pix  = s->hdsp.put_pixels_tab;
2009             op_qpix = s->qdsp.put_qpel_pixels_tab;
2010         } else {
2011             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
2012             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
2013         }
2014
2015         if (s->mv_dir & MV_DIR_FORWARD) {
2016             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 0,
2017                           s->last_picture.f->data,
2018                           op_pix, op_qpix);
2019             op_pix  = s->hdsp.avg_pixels_tab;
2020             op_qpix = s->qdsp.avg_qpel_pixels_tab;
2021         }
2022         if (s->mv_dir & MV_DIR_BACKWARD) {
2023             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 1,
2024                           s->next_picture.f->data,
2025                           op_pix, op_qpix);
2026         }
2027
2028         if (s->avctx->flags & CODEC_FLAG_INTERLACED_DCT) {
2029             int progressive_score, interlaced_score;
2030
2031             s->interlaced_dct = 0;
2032             progressive_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y, wrap_y, 8) +
2033                                 s->mecc.ildct_cmp[0](s, dest_y + wrap_y * 8,
2034                                                      ptr_y + wrap_y * 8,
2035                                                      wrap_y, 8) - 400;
2036
2037             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2038                 progressive_score -= 400;
2039
2040             if (progressive_score > 0) {
2041                 interlaced_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y,
2042                                                         wrap_y * 2, 8) +
2043                                    s->mecc.ildct_cmp[0](s, dest_y + wrap_y,
2044                                                         ptr_y + wrap_y,
2045                                                         wrap_y * 2, 8);
2046
2047                 if (progressive_score > interlaced_score) {
2048                     s->interlaced_dct = 1;
2049
2050                     dct_offset = wrap_y;
2051                     wrap_y <<= 1;
2052                     if (s->chroma_format == CHROMA_422)
2053                         wrap_c <<= 1;
2054                 }
2055             }
2056         }
2057
2058         s->pdsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2059         s->pdsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2060         s->pdsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2061                             dest_y + dct_offset, wrap_y);
2062         s->pdsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2063                             dest_y + dct_offset + 8, wrap_y);
2064
2065         if (s->avctx->flags & CODEC_FLAG_GRAY) {
2066             skip_dct[4] = 1;
2067             skip_dct[5] = 1;
2068         } else {
2069             s->pdsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2070             s->pdsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2071             if (!s->chroma_y_shift) { /* 422 */
2072                 s->pdsp.diff_pixels(s->block[6], ptr_cb + (dct_offset >> 1),
2073                                     dest_cb + (dct_offset >> 1), wrap_c);
2074                 s->pdsp.diff_pixels(s->block[7], ptr_cr + (dct_offset >> 1),
2075                                     dest_cr + (dct_offset >> 1), wrap_c);
2076             }
2077         }
2078         /* pre quantization */
2079         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2080                 2 * s->qscale * s->qscale) {
2081             // FIXME optimize
2082             if (s->mecc.sad[1](NULL, ptr_y, dest_y, wrap_y, 8) < 20 * s->qscale)
2083                 skip_dct[0] = 1;
2084             if (s->mecc.sad[1](NULL, ptr_y + 8, dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2085                 skip_dct[1] = 1;
2086             if (s->mecc.sad[1](NULL, ptr_y + dct_offset, dest_y + dct_offset,
2087                                wrap_y, 8) < 20 * s->qscale)
2088                 skip_dct[2] = 1;
2089             if (s->mecc.sad[1](NULL, ptr_y + dct_offset + 8, dest_y + dct_offset + 8,
2090                                wrap_y, 8) < 20 * s->qscale)
2091                 skip_dct[3] = 1;
2092             if (s->mecc.sad[1](NULL, ptr_cb, dest_cb, wrap_c, 8) < 20 * s->qscale)
2093                 skip_dct[4] = 1;
2094             if (s->mecc.sad[1](NULL, ptr_cr, dest_cr, wrap_c, 8) < 20 * s->qscale)
2095                 skip_dct[5] = 1;
2096             if (!s->chroma_y_shift) { /* 422 */
2097                 if (s->mecc.sad[1](NULL, ptr_cb + (dct_offset >> 1),
2098                                    dest_cb + (dct_offset >> 1),
2099                                    wrap_c, 8) < 20 * s->qscale)
2100                     skip_dct[6] = 1;
2101                 if (s->mecc.sad[1](NULL, ptr_cr + (dct_offset >> 1),
2102                                    dest_cr + (dct_offset >> 1),
2103                                    wrap_c, 8) < 20 * s->qscale)
2104                     skip_dct[7] = 1;
2105             }
2106         }
2107     }
2108
2109     if (s->quantizer_noise_shaping) {
2110         if (!skip_dct[0])
2111             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2112         if (!skip_dct[1])
2113             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2114         if (!skip_dct[2])
2115             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2116         if (!skip_dct[3])
2117             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2118         if (!skip_dct[4])
2119             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2120         if (!skip_dct[5])
2121             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2122         if (!s->chroma_y_shift) { /* 422 */
2123             if (!skip_dct[6])
2124                 get_visual_weight(weight[6], ptr_cb + (dct_offset >> 1),
2125                                   wrap_c);
2126             if (!skip_dct[7])
2127                 get_visual_weight(weight[7], ptr_cr + (dct_offset >> 1),
2128                                   wrap_c);
2129         }
2130         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2131     }
2132
2133     /* DCT & quantize */
2134     assert(s->out_format != FMT_MJPEG || s->qscale == 8);
2135     {
2136         for (i = 0; i < mb_block_count; i++) {
2137             if (!skip_dct[i]) {
2138                 int overflow;
2139                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2140                 // FIXME we could decide to change to quantizer instead of
2141                 // clipping
2142                 // JS: I don't think that would be a good idea it could lower
2143                 //     quality instead of improve it. Just INTRADC clipping
2144                 //     deserves changes in quantizer
2145                 if (overflow)
2146                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2147             } else
2148                 s->block_last_index[i] = -1;
2149         }
2150         if (s->quantizer_noise_shaping) {
2151             for (i = 0; i < mb_block_count; i++) {
2152                 if (!skip_dct[i]) {
2153                     s->block_last_index[i] =
2154                         dct_quantize_refine(s, s->block[i], weight[i],
2155                                             orig[i], i, s->qscale);
2156                 }
2157             }
2158         }
2159
2160         if (s->luma_elim_threshold && !s->mb_intra)
2161             for (i = 0; i < 4; i++)
2162                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2163         if (s->chroma_elim_threshold && !s->mb_intra)
2164             for (i = 4; i < mb_block_count; i++)
2165                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2166
2167         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2168             for (i = 0; i < mb_block_count; i++) {
2169                 if (s->block_last_index[i] == -1)
2170                     s->coded_score[i] = INT_MAX / 256;
2171             }
2172         }
2173     }
2174
2175     if ((s->avctx->flags & CODEC_FLAG_GRAY) && s->mb_intra) {
2176         s->block_last_index[4] =
2177         s->block_last_index[5] = 0;
2178         s->block[4][0] =
2179         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2180     }
2181
2182     // non c quantize code returns incorrect block_last_index FIXME
2183     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2184         for (i = 0; i < mb_block_count; i++) {
2185             int j;
2186             if (s->block_last_index[i] > 0) {
2187                 for (j = 63; j > 0; j--) {
2188                     if (s->block[i][s->intra_scantable.permutated[j]])
2189                         break;
2190                 }
2191                 s->block_last_index[i] = j;
2192             }
2193         }
2194     }
2195
2196     /* huffman encode */
2197     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2198     case AV_CODEC_ID_MPEG1VIDEO:
2199     case AV_CODEC_ID_MPEG2VIDEO:
2200         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2201             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2202         break;
2203     case AV_CODEC_ID_MPEG4:
2204         if (CONFIG_MPEG4_ENCODER)
2205             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2206         break;
2207     case AV_CODEC_ID_MSMPEG4V2:
2208     case AV_CODEC_ID_MSMPEG4V3:
2209     case AV_CODEC_ID_WMV1:
2210         if (CONFIG_MSMPEG4_ENCODER)
2211             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2212         break;
2213     case AV_CODEC_ID_WMV2:
2214         if (CONFIG_WMV2_ENCODER)
2215             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2216         break;
2217     case AV_CODEC_ID_H261:
2218         if (CONFIG_H261_ENCODER)
2219             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2220         break;
2221     case AV_CODEC_ID_H263:
2222     case AV_CODEC_ID_H263P:
2223     case AV_CODEC_ID_FLV1:
2224     case AV_CODEC_ID_RV10:
2225     case AV_CODEC_ID_RV20:
2226         if (CONFIG_H263_ENCODER)
2227             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2228         break;
2229     case AV_CODEC_ID_MJPEG:
2230         if (CONFIG_MJPEG_ENCODER)
2231             ff_mjpeg_encode_mb(s, s->block);
2232         break;
2233     default:
2234         assert(0);
2235     }
2236 }
2237
2238 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2239 {
2240     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
2241     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
2242 }
2243
2244 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2245     int i;
2246
2247     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2248
2249     /* mpeg1 */
2250     d->mb_skip_run= s->mb_skip_run;
2251     for(i=0; i<3; i++)
2252         d->last_dc[i] = s->last_dc[i];
2253
2254     /* statistics */
2255     d->mv_bits= s->mv_bits;
2256     d->i_tex_bits= s->i_tex_bits;
2257     d->p_tex_bits= s->p_tex_bits;
2258     d->i_count= s->i_count;
2259     d->f_count= s->f_count;
2260     d->b_count= s->b_count;
2261     d->skip_count= s->skip_count;
2262     d->misc_bits= s->misc_bits;
2263     d->last_bits= 0;
2264
2265     d->mb_skipped= 0;
2266     d->qscale= s->qscale;
2267     d->dquant= s->dquant;
2268
2269     d->esc3_level_length= s->esc3_level_length;
2270 }
2271
2272 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2273     int i;
2274
2275     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2276     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2277
2278     /* mpeg1 */
2279     d->mb_skip_run= s->mb_skip_run;
2280     for(i=0; i<3; i++)
2281         d->last_dc[i] = s->last_dc[i];
2282
2283     /* statistics */
2284     d->mv_bits= s->mv_bits;
2285     d->i_tex_bits= s->i_tex_bits;
2286     d->p_tex_bits= s->p_tex_bits;
2287     d->i_count= s->i_count;
2288     d->f_count= s->f_count;
2289     d->b_count= s->b_count;
2290     d->skip_count= s->skip_count;
2291     d->misc_bits= s->misc_bits;
2292
2293     d->mb_intra= s->mb_intra;
2294     d->mb_skipped= s->mb_skipped;
2295     d->mv_type= s->mv_type;
2296     d->mv_dir= s->mv_dir;
2297     d->pb= s->pb;
2298     if(s->data_partitioning){
2299         d->pb2= s->pb2;
2300         d->tex_pb= s->tex_pb;
2301     }
2302     d->block= s->block;
2303     for(i=0; i<8; i++)
2304         d->block_last_index[i]= s->block_last_index[i];
2305     d->interlaced_dct= s->interlaced_dct;
2306     d->qscale= s->qscale;
2307
2308     d->esc3_level_length= s->esc3_level_length;
2309 }
2310
2311 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2312                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2313                            int *dmin, int *next_block, int motion_x, int motion_y)
2314 {
2315     int score;
2316     uint8_t *dest_backup[3];
2317
2318     copy_context_before_encode(s, backup, type);
2319
2320     s->block= s->blocks[*next_block];
2321     s->pb= pb[*next_block];
2322     if(s->data_partitioning){
2323         s->pb2   = pb2   [*next_block];
2324         s->tex_pb= tex_pb[*next_block];
2325     }
2326
2327     if(*next_block){
2328         memcpy(dest_backup, s->dest, sizeof(s->dest));
2329         s->dest[0] = s->rd_scratchpad;
2330         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
2331         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
2332         assert(s->linesize >= 32); //FIXME
2333     }
2334
2335     encode_mb(s, motion_x, motion_y);
2336
2337     score= put_bits_count(&s->pb);
2338     if(s->data_partitioning){
2339         score+= put_bits_count(&s->pb2);
2340         score+= put_bits_count(&s->tex_pb);
2341     }
2342
2343     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2344         ff_mpv_decode_mb(s, s->block);
2345
2346         score *= s->lambda2;
2347         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2348     }
2349
2350     if(*next_block){
2351         memcpy(s->dest, dest_backup, sizeof(s->dest));
2352     }
2353
2354     if(score<*dmin){
2355         *dmin= score;
2356         *next_block^=1;
2357
2358         copy_context_after_encode(best, s, type);
2359     }
2360 }
2361
2362 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2363     uint32_t *sq = ff_square_tab + 256;
2364     int acc=0;
2365     int x,y;
2366
2367     if(w==16 && h==16)
2368         return s->mecc.sse[0](NULL, src1, src2, stride, 16);
2369     else if(w==8 && h==8)
2370         return s->mecc.sse[1](NULL, src1, src2, stride, 8);
2371
2372     for(y=0; y<h; y++){
2373         for(x=0; x<w; x++){
2374             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2375         }
2376     }
2377
2378     assert(acc>=0);
2379
2380     return acc;
2381 }
2382
2383 static int sse_mb(MpegEncContext *s){
2384     int w= 16;
2385     int h= 16;
2386
2387     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2388     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2389
2390     if(w==16 && h==16)
2391       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2392         return s->mecc.nsse[0](s, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2393                s->mecc.nsse[1](s, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2394                s->mecc.nsse[1](s, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2395       }else{
2396         return s->mecc.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2397                s->mecc.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2398                s->mecc.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2399       }
2400     else
2401         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2402                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2403                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2404 }
2405
2406 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2407     MpegEncContext *s= *(void**)arg;
2408
2409
2410     s->me.pre_pass=1;
2411     s->me.dia_size= s->avctx->pre_dia_size;
2412     s->first_slice_line=1;
2413     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2414         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2415             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2416         }
2417         s->first_slice_line=0;
2418     }
2419
2420     s->me.pre_pass=0;
2421
2422     return 0;
2423 }
2424
2425 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2426     MpegEncContext *s= *(void**)arg;
2427
2428     s->me.dia_size= s->avctx->dia_size;
2429     s->first_slice_line=1;
2430     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2431         s->mb_x=0; //for block init below
2432         ff_init_block_index(s);
2433         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2434             s->block_index[0]+=2;
2435             s->block_index[1]+=2;
2436             s->block_index[2]+=2;
2437             s->block_index[3]+=2;
2438
2439             /* compute motion vector & mb_type and store in context */
2440             if(s->pict_type==AV_PICTURE_TYPE_B)
2441                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2442             else
2443                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2444         }
2445         s->first_slice_line=0;
2446     }
2447     return 0;
2448 }
2449
2450 static int mb_var_thread(AVCodecContext *c, void *arg){
2451     MpegEncContext *s= *(void**)arg;
2452     int mb_x, mb_y;
2453
2454     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2455         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2456             int xx = mb_x * 16;
2457             int yy = mb_y * 16;
2458             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2459             int varc;
2460             int sum = s->mpvencdsp.pix_sum(pix, s->linesize);
2461
2462             varc = (s->mpvencdsp.pix_norm1(pix, s->linesize) -
2463                     (((unsigned) sum * sum) >> 8) + 500 + 128) >> 8;
2464
2465             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2466             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2467             s->me.mb_var_sum_temp    += varc;
2468         }
2469     }
2470     return 0;
2471 }
2472
2473 static void write_slice_end(MpegEncContext *s){
2474     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2475         if(s->partitioned_frame){
2476             ff_mpeg4_merge_partitions(s);
2477         }
2478
2479         ff_mpeg4_stuffing(&s->pb);
2480     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2481         ff_mjpeg_encode_stuffing(&s->pb);
2482     }
2483
2484     avpriv_align_put_bits(&s->pb);
2485     flush_put_bits(&s->pb);
2486
2487     if ((s->avctx->flags & CODEC_FLAG_PASS1) && !s->partitioned_frame)
2488         s->misc_bits+= get_bits_diff(s);
2489 }
2490
2491 static void write_mb_info(MpegEncContext *s)
2492 {
2493     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2494     int offset = put_bits_count(&s->pb);
2495     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2496     int gobn = s->mb_y / s->gob_index;
2497     int pred_x, pred_y;
2498     if (CONFIG_H263_ENCODER)
2499         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2500     bytestream_put_le32(&ptr, offset);
2501     bytestream_put_byte(&ptr, s->qscale);
2502     bytestream_put_byte(&ptr, gobn);
2503     bytestream_put_le16(&ptr, mba);
2504     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2505     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2506     /* 4MV not implemented */
2507     bytestream_put_byte(&ptr, 0); /* hmv2 */
2508     bytestream_put_byte(&ptr, 0); /* vmv2 */
2509 }
2510
2511 static void update_mb_info(MpegEncContext *s, int startcode)
2512 {
2513     if (!s->mb_info)
2514         return;
2515     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2516         s->mb_info_size += 12;
2517         s->prev_mb_info = s->last_mb_info;
2518     }
2519     if (startcode) {
2520         s->prev_mb_info = put_bits_count(&s->pb)/8;
2521         /* This might have incremented mb_info_size above, and we return without
2522          * actually writing any info into that slot yet. But in that case,
2523          * this will be called again at the start of the after writing the
2524          * start code, actually writing the mb info. */
2525         return;
2526     }
2527
2528     s->last_mb_info = put_bits_count(&s->pb)/8;
2529     if (!s->mb_info_size)
2530         s->mb_info_size += 12;
2531     write_mb_info(s);
2532 }
2533
2534 static int encode_thread(AVCodecContext *c, void *arg){
2535     MpegEncContext *s= *(void**)arg;
2536     int mb_x, mb_y, pdif = 0;
2537     int chr_h= 16>>s->chroma_y_shift;
2538     int i, j;
2539     MpegEncContext best_s = { 0 }, backup_s;
2540     uint8_t bit_buf[2][MAX_MB_BYTES];
2541     uint8_t bit_buf2[2][MAX_MB_BYTES];
2542     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2543     PutBitContext pb[2], pb2[2], tex_pb[2];
2544
2545     for(i=0; i<2; i++){
2546         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2547         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2548         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2549     }
2550
2551     s->last_bits= put_bits_count(&s->pb);
2552     s->mv_bits=0;
2553     s->misc_bits=0;
2554     s->i_tex_bits=0;
2555     s->p_tex_bits=0;
2556     s->i_count=0;
2557     s->f_count=0;
2558     s->b_count=0;
2559     s->skip_count=0;
2560
2561     for(i=0; i<3; i++){
2562         /* init last dc values */
2563         /* note: quant matrix value (8) is implied here */
2564         s->last_dc[i] = 128 << s->intra_dc_precision;
2565
2566         s->current_picture.f->error[i] = 0;
2567     }
2568     s->mb_skip_run = 0;
2569     memset(s->last_mv, 0, sizeof(s->last_mv));
2570
2571     s->last_mv_dir = 0;
2572
2573     switch(s->codec_id){
2574     case AV_CODEC_ID_H263:
2575     case AV_CODEC_ID_H263P:
2576     case AV_CODEC_ID_FLV1:
2577         if (CONFIG_H263_ENCODER)
2578             s->gob_index = H263_GOB_HEIGHT(s->height);
2579         break;
2580     case AV_CODEC_ID_MPEG4:
2581         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2582             ff_mpeg4_init_partitions(s);
2583         break;
2584     }
2585
2586     s->resync_mb_x=0;
2587     s->resync_mb_y=0;
2588     s->first_slice_line = 1;
2589     s->ptr_lastgob = s->pb.buf;
2590     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2591         s->mb_x=0;
2592         s->mb_y= mb_y;
2593
2594         ff_set_qscale(s, s->qscale);
2595         ff_init_block_index(s);
2596
2597         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2598             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2599             int mb_type= s->mb_type[xy];
2600 //            int d;
2601             int dmin= INT_MAX;
2602             int dir;
2603
2604             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2605                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2606                 return -1;
2607             }
2608             if(s->data_partitioning){
2609                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2610                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2611                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2612                     return -1;
2613                 }
2614             }
2615
2616             s->mb_x = mb_x;
2617             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2618             ff_update_block_index(s);
2619
2620             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
2621                 ff_h261_reorder_mb_index(s);
2622                 xy= s->mb_y*s->mb_stride + s->mb_x;
2623                 mb_type= s->mb_type[xy];
2624             }
2625
2626             /* write gob / video packet header  */
2627             if(s->rtp_mode){
2628                 int current_packet_size, is_gob_start;
2629
2630                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2631
2632                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2633
2634                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2635
2636                 switch(s->codec_id){
2637                 case AV_CODEC_ID_H263:
2638                 case AV_CODEC_ID_H263P:
2639                     if(!s->h263_slice_structured)
2640                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2641                     break;
2642                 case AV_CODEC_ID_MPEG2VIDEO:
2643                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2644                 case AV_CODEC_ID_MPEG1VIDEO:
2645                     if(s->mb_skip_run) is_gob_start=0;
2646                     break;
2647                 }
2648
2649                 if(is_gob_start){
2650                     if(s->start_mb_y != mb_y || mb_x!=0){
2651                         write_slice_end(s);
2652
2653                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
2654                             ff_mpeg4_init_partitions(s);
2655                         }
2656                     }
2657
2658                     assert((put_bits_count(&s->pb)&7) == 0);
2659                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2660
2661                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
2662                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2663                         int d = 100 / s->error_rate;
2664                         if(r % d == 0){
2665                             current_packet_size=0;
2666                             s->pb.buf_ptr= s->ptr_lastgob;
2667                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2668                         }
2669                     }
2670
2671                     if (s->avctx->rtp_callback){
2672                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2673                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2674                     }
2675                     update_mb_info(s, 1);
2676
2677                     switch(s->codec_id){
2678                     case AV_CODEC_ID_MPEG4:
2679                         if (CONFIG_MPEG4_ENCODER) {
2680                             ff_mpeg4_encode_video_packet_header(s);
2681                             ff_mpeg4_clean_buffers(s);
2682                         }
2683                     break;
2684                     case AV_CODEC_ID_MPEG1VIDEO:
2685                     case AV_CODEC_ID_MPEG2VIDEO:
2686                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2687                             ff_mpeg1_encode_slice_header(s);
2688                             ff_mpeg1_clean_buffers(s);
2689                         }
2690                     break;
2691                     case AV_CODEC_ID_H263:
2692                     case AV_CODEC_ID_H263P:
2693                         if (CONFIG_H263_ENCODER)
2694                             ff_h263_encode_gob_header(s, mb_y);
2695                     break;
2696                     }
2697
2698                     if (s->avctx->flags & CODEC_FLAG_PASS1) {
2699                         int bits= put_bits_count(&s->pb);
2700                         s->misc_bits+= bits - s->last_bits;
2701                         s->last_bits= bits;
2702                     }
2703
2704                     s->ptr_lastgob += current_packet_size;
2705                     s->first_slice_line=1;
2706                     s->resync_mb_x=mb_x;
2707                     s->resync_mb_y=mb_y;
2708                 }
2709             }
2710
2711             if(  (s->resync_mb_x   == s->mb_x)
2712                && s->resync_mb_y+1 == s->mb_y){
2713                 s->first_slice_line=0;
2714             }
2715
2716             s->mb_skipped=0;
2717             s->dquant=0; //only for QP_RD
2718
2719             update_mb_info(s, 0);
2720
2721             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
2722                 int next_block=0;
2723                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2724
2725                 copy_context_before_encode(&backup_s, s, -1);
2726                 backup_s.pb= s->pb;
2727                 best_s.data_partitioning= s->data_partitioning;
2728                 best_s.partitioned_frame= s->partitioned_frame;
2729                 if(s->data_partitioning){
2730                     backup_s.pb2= s->pb2;
2731                     backup_s.tex_pb= s->tex_pb;
2732                 }
2733
2734                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2735                     s->mv_dir = MV_DIR_FORWARD;
2736                     s->mv_type = MV_TYPE_16X16;
2737                     s->mb_intra= 0;
2738                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2739                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2740                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2741                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2742                 }
2743                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2744                     s->mv_dir = MV_DIR_FORWARD;
2745                     s->mv_type = MV_TYPE_FIELD;
2746                     s->mb_intra= 0;
2747                     for(i=0; i<2; i++){
2748                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2749                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2750                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2751                     }
2752                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2753                                  &dmin, &next_block, 0, 0);
2754                 }
2755                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2756                     s->mv_dir = MV_DIR_FORWARD;
2757                     s->mv_type = MV_TYPE_16X16;
2758                     s->mb_intra= 0;
2759                     s->mv[0][0][0] = 0;
2760                     s->mv[0][0][1] = 0;
2761                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2762                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2763                 }
2764                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2765                     s->mv_dir = MV_DIR_FORWARD;
2766                     s->mv_type = MV_TYPE_8X8;
2767                     s->mb_intra= 0;
2768                     for(i=0; i<4; i++){
2769                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2770                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2771                     }
2772                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2773                                  &dmin, &next_block, 0, 0);
2774                 }
2775                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2776                     s->mv_dir = MV_DIR_FORWARD;
2777                     s->mv_type = MV_TYPE_16X16;
2778                     s->mb_intra= 0;
2779                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2780                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2781                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2782                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2783                 }
2784                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2785                     s->mv_dir = MV_DIR_BACKWARD;
2786                     s->mv_type = MV_TYPE_16X16;
2787                     s->mb_intra= 0;
2788                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2789                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2790                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2791                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2792                 }
2793                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2794                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2795                     s->mv_type = MV_TYPE_16X16;
2796                     s->mb_intra= 0;
2797                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2798                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2799                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2800                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2801                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2802                                  &dmin, &next_block, 0, 0);
2803                 }
2804                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2805                     s->mv_dir = MV_DIR_FORWARD;
2806                     s->mv_type = MV_TYPE_FIELD;
2807                     s->mb_intra= 0;
2808                     for(i=0; i<2; i++){
2809                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2810                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2811                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2812                     }
2813                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2814                                  &dmin, &next_block, 0, 0);
2815                 }
2816                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2817                     s->mv_dir = MV_DIR_BACKWARD;
2818                     s->mv_type = MV_TYPE_FIELD;
2819                     s->mb_intra= 0;
2820                     for(i=0; i<2; i++){
2821                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2822                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2823                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2824                     }
2825                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2826                                  &dmin, &next_block, 0, 0);
2827                 }
2828                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2829                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2830                     s->mv_type = MV_TYPE_FIELD;
2831                     s->mb_intra= 0;
2832                     for(dir=0; dir<2; dir++){
2833                         for(i=0; i<2; i++){
2834                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2835                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2836                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2837                         }
2838                     }
2839                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2840                                  &dmin, &next_block, 0, 0);
2841                 }
2842                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2843                     s->mv_dir = 0;
2844                     s->mv_type = MV_TYPE_16X16;
2845                     s->mb_intra= 1;
2846                     s->mv[0][0][0] = 0;
2847                     s->mv[0][0][1] = 0;
2848                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2849                                  &dmin, &next_block, 0, 0);
2850                     if(s->h263_pred || s->h263_aic){
2851                         if(best_s.mb_intra)
2852                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2853                         else
2854                             ff_clean_intra_table_entries(s); //old mode?
2855                     }
2856                 }
2857
2858                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
2859                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2860                         const int last_qp= backup_s.qscale;
2861                         int qpi, qp, dc[6];
2862                         int16_t ac[6][16];
2863                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2864                         static const int dquant_tab[4]={-1,1,-2,2};
2865
2866                         assert(backup_s.dquant == 0);
2867
2868                         //FIXME intra
2869                         s->mv_dir= best_s.mv_dir;
2870                         s->mv_type = MV_TYPE_16X16;
2871                         s->mb_intra= best_s.mb_intra;
2872                         s->mv[0][0][0] = best_s.mv[0][0][0];
2873                         s->mv[0][0][1] = best_s.mv[0][0][1];
2874                         s->mv[1][0][0] = best_s.mv[1][0][0];
2875                         s->mv[1][0][1] = best_s.mv[1][0][1];
2876
2877                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
2878                         for(; qpi<4; qpi++){
2879                             int dquant= dquant_tab[qpi];
2880                             qp= last_qp + dquant;
2881                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2882                                 continue;
2883                             backup_s.dquant= dquant;
2884                             if(s->mb_intra && s->dc_val[0]){
2885                                 for(i=0; i<6; i++){
2886                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2887                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(int16_t)*16);
2888                                 }
2889                             }
2890
2891                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2892                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2893                             if(best_s.qscale != qp){
2894                                 if(s->mb_intra && s->dc_val[0]){
2895                                     for(i=0; i<6; i++){
2896                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2897                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(int16_t)*16);
2898                                     }
2899                                 }
2900                             }
2901                         }
2902                     }
2903                 }
2904                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2905                     int mx= s->b_direct_mv_table[xy][0];
2906                     int my= s->b_direct_mv_table[xy][1];
2907
2908                     backup_s.dquant = 0;
2909                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2910                     s->mb_intra= 0;
2911                     ff_mpeg4_set_direct_mv(s, mx, my);
2912                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2913                                  &dmin, &next_block, mx, my);
2914                 }
2915                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2916                     backup_s.dquant = 0;
2917                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2918                     s->mb_intra= 0;
2919                     ff_mpeg4_set_direct_mv(s, 0, 0);
2920                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2921                                  &dmin, &next_block, 0, 0);
2922                 }
2923                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
2924                     int coded=0;
2925                     for(i=0; i<6; i++)
2926                         coded |= s->block_last_index[i];
2927                     if(coded){
2928                         int mx,my;
2929                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2930                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
2931                             mx=my=0; //FIXME find the one we actually used
2932                             ff_mpeg4_set_direct_mv(s, mx, my);
2933                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2934                             mx= s->mv[1][0][0];
2935                             my= s->mv[1][0][1];
2936                         }else{
2937                             mx= s->mv[0][0][0];
2938                             my= s->mv[0][0][1];
2939                         }
2940
2941                         s->mv_dir= best_s.mv_dir;
2942                         s->mv_type = best_s.mv_type;
2943                         s->mb_intra= 0;
2944 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2945                         s->mv[0][0][1] = best_s.mv[0][0][1];
2946                         s->mv[1][0][0] = best_s.mv[1][0][0];
2947                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2948                         backup_s.dquant= 0;
2949                         s->skipdct=1;
2950                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2951                                         &dmin, &next_block, mx, my);
2952                         s->skipdct=0;
2953                     }
2954                 }
2955
2956                 s->current_picture.qscale_table[xy] = best_s.qscale;
2957
2958                 copy_context_after_encode(s, &best_s, -1);
2959
2960                 pb_bits_count= put_bits_count(&s->pb);
2961                 flush_put_bits(&s->pb);
2962                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2963                 s->pb= backup_s.pb;
2964
2965                 if(s->data_partitioning){
2966                     pb2_bits_count= put_bits_count(&s->pb2);
2967                     flush_put_bits(&s->pb2);
2968                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2969                     s->pb2= backup_s.pb2;
2970
2971                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2972                     flush_put_bits(&s->tex_pb);
2973                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2974                     s->tex_pb= backup_s.tex_pb;
2975                 }
2976                 s->last_bits= put_bits_count(&s->pb);
2977
2978                 if (CONFIG_H263_ENCODER &&
2979                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2980                     ff_h263_update_motion_val(s);
2981
2982                 if(next_block==0){ //FIXME 16 vs linesize16
2983                     s->hdsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
2984                     s->hdsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2985                     s->hdsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2986                 }
2987
2988                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
2989                     ff_mpv_decode_mb(s, s->block);
2990             } else {
2991                 int motion_x = 0, motion_y = 0;
2992                 s->mv_type=MV_TYPE_16X16;
2993                 // only one MB-Type possible
2994
2995                 switch(mb_type){
2996                 case CANDIDATE_MB_TYPE_INTRA:
2997                     s->mv_dir = 0;
2998                     s->mb_intra= 1;
2999                     motion_x= s->mv[0][0][0] = 0;
3000                     motion_y= s->mv[0][0][1] = 0;
3001                     break;
3002                 case CANDIDATE_MB_TYPE_INTER:
3003                     s->mv_dir = MV_DIR_FORWARD;
3004                     s->mb_intra= 0;
3005                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
3006                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
3007                     break;
3008                 case CANDIDATE_MB_TYPE_INTER_I:
3009                     s->mv_dir = MV_DIR_FORWARD;
3010                     s->mv_type = MV_TYPE_FIELD;
3011                     s->mb_intra= 0;
3012                     for(i=0; i<2; i++){
3013                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3014                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3015                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3016                     }
3017                     break;
3018                 case CANDIDATE_MB_TYPE_INTER4V:
3019                     s->mv_dir = MV_DIR_FORWARD;
3020                     s->mv_type = MV_TYPE_8X8;
3021                     s->mb_intra= 0;
3022                     for(i=0; i<4; i++){
3023                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3024                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3025                     }
3026                     break;
3027                 case CANDIDATE_MB_TYPE_DIRECT:
3028                     if (CONFIG_MPEG4_ENCODER) {
3029                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3030                         s->mb_intra= 0;
3031                         motion_x=s->b_direct_mv_table[xy][0];
3032                         motion_y=s->b_direct_mv_table[xy][1];
3033                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
3034                     }
3035                     break;
3036                 case CANDIDATE_MB_TYPE_DIRECT0:
3037                     if (CONFIG_MPEG4_ENCODER) {
3038                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3039                         s->mb_intra= 0;
3040                         ff_mpeg4_set_direct_mv(s, 0, 0);
3041                     }
3042                     break;
3043                 case CANDIDATE_MB_TYPE_BIDIR:
3044                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3045                     s->mb_intra= 0;
3046                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3047                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3048                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3049                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3050                     break;
3051                 case CANDIDATE_MB_TYPE_BACKWARD:
3052                     s->mv_dir = MV_DIR_BACKWARD;
3053                     s->mb_intra= 0;
3054                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3055                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3056                     break;
3057                 case CANDIDATE_MB_TYPE_FORWARD:
3058                     s->mv_dir = MV_DIR_FORWARD;
3059                     s->mb_intra= 0;
3060                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3061                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3062                     break;
3063                 case CANDIDATE_MB_TYPE_FORWARD_I:
3064                     s->mv_dir = MV_DIR_FORWARD;
3065                     s->mv_type = MV_TYPE_FIELD;
3066                     s->mb_intra= 0;
3067                     for(i=0; i<2; i++){
3068                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3069                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3070                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3071                     }
3072                     break;
3073                 case CANDIDATE_MB_TYPE_BACKWARD_I:
3074                     s->mv_dir = MV_DIR_BACKWARD;
3075                     s->mv_type = MV_TYPE_FIELD;
3076                     s->mb_intra= 0;
3077                     for(i=0; i<2; i++){
3078                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3079                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3080                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3081                     }
3082                     break;
3083                 case CANDIDATE_MB_TYPE_BIDIR_I:
3084                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3085                     s->mv_type = MV_TYPE_FIELD;
3086                     s->mb_intra= 0;
3087                     for(dir=0; dir<2; dir++){
3088                         for(i=0; i<2; i++){
3089                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3090                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3091                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3092                         }
3093                     }
3094                     break;
3095                 default:
3096                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
3097                 }
3098
3099                 encode_mb(s, motion_x, motion_y);
3100
3101                 // RAL: Update last macroblock type
3102                 s->last_mv_dir = s->mv_dir;
3103
3104                 if (CONFIG_H263_ENCODER &&
3105                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3106                     ff_h263_update_motion_val(s);
3107
3108                 ff_mpv_decode_mb(s, s->block);
3109             }
3110
3111             /* clean the MV table in IPS frames for direct mode in B frames */
3112             if(s->mb_intra /* && I,P,S_TYPE */){
3113                 s->p_mv_table[xy][0]=0;
3114                 s->p_mv_table[xy][1]=0;
3115             }
3116
3117             if (s->avctx->flags & CODEC_FLAG_PSNR) {
3118                 int w= 16;
3119                 int h= 16;
3120
3121                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
3122                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
3123
3124                 s->current_picture.f->error[0] += sse(
3125                     s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
3126                     s->dest[0], w, h, s->linesize);
3127                 s->current_picture.f->error[1] += sse(
3128                     s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3129                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3130                 s->current_picture.f->error[2] += sse(
3131                     s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3132                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3133             }
3134             if(s->loop_filter){
3135                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
3136                     ff_h263_loop_filter(s);
3137             }
3138             ff_dlog(s->avctx, "MB %d %d bits\n",
3139                     s->mb_x + s->mb_y * s->mb_stride, put_bits_count(&s->pb));
3140         }
3141     }
3142
3143     //not beautiful here but we must write it before flushing so it has to be here
3144     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
3145         ff_msmpeg4_encode_ext_header(s);
3146
3147     write_slice_end(s);
3148
3149     /* Send the last GOB if RTP */
3150     if (s->avctx->rtp_callback) {
3151         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
3152         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
3153         /* Call the RTP callback to send the last GOB */
3154         emms_c();
3155         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
3156     }
3157
3158     return 0;
3159 }
3160
3161 #define MERGE(field) dst->field += src->field; src->field=0
3162 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
3163     MERGE(me.scene_change_score);
3164     MERGE(me.mc_mb_var_sum_temp);
3165     MERGE(me.mb_var_sum_temp);
3166 }
3167
3168 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
3169     int i;
3170
3171     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
3172     MERGE(dct_count[1]);
3173     MERGE(mv_bits);
3174     MERGE(i_tex_bits);
3175     MERGE(p_tex_bits);
3176     MERGE(i_count);
3177     MERGE(f_count);
3178     MERGE(b_count);
3179     MERGE(skip_count);
3180     MERGE(misc_bits);
3181     MERGE(er.error_count);
3182     MERGE(padding_bug_score);
3183     MERGE(current_picture.f->error[0]);
3184     MERGE(current_picture.f->error[1]);
3185     MERGE(current_picture.f->error[2]);
3186
3187     if(dst->avctx->noise_reduction){
3188         for(i=0; i<64; i++){
3189             MERGE(dct_error_sum[0][i]);
3190             MERGE(dct_error_sum[1][i]);
3191         }
3192     }
3193
3194     assert(put_bits_count(&src->pb) % 8 ==0);
3195     assert(put_bits_count(&dst->pb) % 8 ==0);
3196     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3197     flush_put_bits(&dst->pb);
3198 }
3199
3200 static int estimate_qp(MpegEncContext *s, int dry_run){
3201     if (s->next_lambda){
3202         s->current_picture_ptr->f->quality =
3203         s->current_picture.f->quality = s->next_lambda;
3204         if(!dry_run) s->next_lambda= 0;
3205     } else if (!s->fixed_qscale) {
3206         s->current_picture_ptr->f->quality =
3207         s->current_picture.f->quality = ff_rate_estimate_qscale(s, dry_run);
3208         if (s->current_picture.f->quality < 0)
3209             return -1;
3210     }
3211
3212     if(s->adaptive_quant){
3213         switch(s->codec_id){
3214         case AV_CODEC_ID_MPEG4:
3215             if (CONFIG_MPEG4_ENCODER)
3216                 ff_clean_mpeg4_qscales(s);
3217             break;
3218         case AV_CODEC_ID_H263:
3219         case AV_CODEC_ID_H263P:
3220         case AV_CODEC_ID_FLV1:
3221             if (CONFIG_H263_ENCODER)
3222                 ff_clean_h263_qscales(s);
3223             break;
3224         default:
3225             ff_init_qscale_tab(s);
3226         }
3227
3228         s->lambda= s->lambda_table[0];
3229         //FIXME broken
3230     }else
3231         s->lambda = s->current_picture.f->quality;
3232     update_qscale(s);
3233     return 0;
3234 }
3235
3236 /* must be called before writing the header */
3237 static void set_frame_distances(MpegEncContext * s){
3238     assert(s->current_picture_ptr->f->pts != AV_NOPTS_VALUE);
3239     s->time = s->current_picture_ptr->f->pts * s->avctx->time_base.num;
3240
3241     if(s->pict_type==AV_PICTURE_TYPE_B){
3242         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3243         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3244     }else{
3245         s->pp_time= s->time - s->last_non_b_time;
3246         s->last_non_b_time= s->time;
3247         assert(s->picture_number==0 || s->pp_time > 0);
3248     }
3249 }
3250
3251 static int encode_picture(MpegEncContext *s, int picture_number)
3252 {
3253     int i, ret;
3254     int bits;
3255     int context_count = s->slice_context_count;
3256
3257     s->picture_number = picture_number;
3258
3259     /* Reset the average MB variance */
3260     s->me.mb_var_sum_temp    =
3261     s->me.mc_mb_var_sum_temp = 0;
3262
3263     /* we need to initialize some time vars before we can encode b-frames */
3264     // RAL: Condition added for MPEG1VIDEO
3265     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3266         set_frame_distances(s);
3267     if(CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4)
3268         ff_set_mpeg4_time(s);
3269
3270     s->me.scene_change_score=0;
3271
3272 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3273
3274     if(s->pict_type==AV_PICTURE_TYPE_I){
3275         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3276         else                        s->no_rounding=0;
3277     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3278         if(s->flipflop_rounding || s->codec_id == AV_CODEC_ID_H263P || s->codec_id == AV_CODEC_ID_MPEG4)
3279             s->no_rounding ^= 1;
3280     }
3281
3282     if (s->avctx->flags & CODEC_FLAG_PASS2) {
3283         if (estimate_qp(s,1) < 0)
3284             return -1;
3285         ff_get_2pass_fcode(s);
3286     } else if (!(s->avctx->flags & CODEC_FLAG_QSCALE)) {
3287         if(s->pict_type==AV_PICTURE_TYPE_B)
3288             s->lambda= s->last_lambda_for[s->pict_type];
3289         else
3290             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3291         update_qscale(s);
3292     }
3293
3294     s->mb_intra=0; //for the rate distortion & bit compare functions
3295     for(i=1; i<context_count; i++){
3296         ret = ff_update_duplicate_context(s->thread_context[i], s);
3297         if (ret < 0)
3298             return ret;
3299     }
3300
3301     if(ff_init_me(s)<0)
3302         return -1;
3303
3304     /* Estimate motion for every MB */
3305     if(s->pict_type != AV_PICTURE_TYPE_I){
3306         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
3307         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
3308         if (s->pict_type != AV_PICTURE_TYPE_B) {
3309             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
3310                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3311             }
3312         }
3313
3314         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3315     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3316         /* I-Frame */
3317         for(i=0; i<s->mb_stride*s->mb_height; i++)
3318             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3319
3320         if(!s->fixed_qscale){
3321             /* finding spatial complexity for I-frame rate control */
3322             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3323         }
3324     }
3325     for(i=1; i<context_count; i++){
3326         merge_context_after_me(s, s->thread_context[i]);
3327     }
3328     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3329     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3330     emms_c();
3331
3332     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
3333         s->pict_type= AV_PICTURE_TYPE_I;
3334         for(i=0; i<s->mb_stride*s->mb_height; i++)
3335             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3336         ff_dlog(s, "Scene change detected, encoding as I Frame %d %d\n",
3337                 s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3338     }
3339
3340     if(!s->umvplus){
3341         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3342             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3343
3344             if (s->avctx->flags & CODEC_FLAG_INTERLACED_ME) {
3345                 int a,b;
3346                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3347                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3348                 s->f_code= FFMAX3(s->f_code, a, b);
3349             }
3350
3351             ff_fix_long_p_mvs(s);
3352             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3353             if (s->avctx->flags & CODEC_FLAG_INTERLACED_ME) {
3354                 int j;
3355                 for(i=0; i<2; i++){
3356                     for(j=0; j<2; j++)
3357                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3358                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3359                 }
3360             }
3361         }
3362
3363         if(s->pict_type==AV_PICTURE_TYPE_B){
3364             int a, b;
3365
3366             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3367             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3368             s->f_code = FFMAX(a, b);
3369
3370             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3371             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3372             s->b_code = FFMAX(a, b);
3373
3374             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3375             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3376             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3377             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3378             if (s->avctx->flags & CODEC_FLAG_INTERLACED_ME) {
3379                 int dir, j;
3380                 for(dir=0; dir<2; dir++){
3381                     for(i=0; i<2; i++){
3382                         for(j=0; j<2; j++){
3383                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3384                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3385                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3386                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3387                         }
3388                     }
3389                 }
3390             }
3391         }
3392     }
3393
3394     if (estimate_qp(s, 0) < 0)
3395         return -1;
3396
3397     if (s->qscale < 3 && s->max_qcoeff <= 128 &&
3398         s->pict_type == AV_PICTURE_TYPE_I &&
3399         !(s->avctx->flags & CODEC_FLAG_QSCALE))
3400         s->qscale= 3; //reduce clipping problems
3401
3402     if (s->out_format == FMT_MJPEG) {
3403         /* for mjpeg, we do include qscale in the matrix */
3404         for(i=1;i<64;i++){
3405             int j = s->idsp.idct_permutation[i];
3406
3407             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
3408         }
3409         s->y_dc_scale_table=
3410         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3411         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3412         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
3413                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3414         s->qscale= 8;
3415     }
3416
3417     //FIXME var duplication
3418     s->current_picture_ptr->f->key_frame =
3419     s->current_picture.f->key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3420     s->current_picture_ptr->f->pict_type =
3421     s->current_picture.f->pict_type = s->pict_type;
3422
3423     if (s->current_picture.f->key_frame)
3424         s->picture_in_gop_number=0;
3425
3426     s->last_bits= put_bits_count(&s->pb);
3427     switch(s->out_format) {
3428     case FMT_MJPEG:
3429         if (CONFIG_MJPEG_ENCODER)
3430             ff_mjpeg_encode_picture_header(s->avctx, &s->pb, &s->intra_scantable,
3431                                            s->intra_matrix);
3432         break;
3433     case FMT_H261:
3434         if (CONFIG_H261_ENCODER)
3435             ff_h261_encode_picture_header(s, picture_number);
3436         break;
3437     case FMT_H263:
3438         if (CONFIG_WMV2_ENCODER && s->codec_id == AV_CODEC_ID_WMV2)
3439             ff_wmv2_encode_picture_header(s, picture_number);
3440         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3441             ff_msmpeg4_encode_picture_header(s, picture_number);
3442         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
3443             ff_mpeg4_encode_picture_header(s, picture_number);
3444         else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10) {
3445             ret = ff_rv10_encode_picture_header(s, picture_number);
3446             if (ret < 0)
3447                 return ret;
3448         }
3449         else if (CONFIG_RV20_ENCODER && s->codec_id == AV_CODEC_ID_RV20)
3450             ff_rv20_encode_picture_header(s, picture_number);
3451         else if (CONFIG_FLV_ENCODER && s->codec_id == AV_CODEC_ID_FLV1)
3452             ff_flv_encode_picture_header(s, picture_number);
3453         else if (CONFIG_H263_ENCODER)
3454             ff_h263_encode_picture_header(s, picture_number);
3455         break;
3456     case FMT_MPEG1:
3457         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
3458             ff_mpeg1_encode_picture_header(s, picture_number);
3459         break;
3460     default:
3461         assert(0);
3462     }
3463     bits= put_bits_count(&s->pb);
3464     s->header_bits= bits - s->last_bits;
3465
3466     for(i=1; i<context_count; i++){
3467         update_duplicate_context_after_me(s->thread_context[i], s);
3468     }
3469     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3470     for(i=1; i<context_count; i++){
3471         merge_context_after_encode(s, s->thread_context[i]);
3472     }
3473     emms_c();
3474     return 0;
3475 }
3476
3477 static void denoise_dct_c(MpegEncContext *s, int16_t *block){
3478     const int intra= s->mb_intra;
3479     int i;
3480
3481     s->dct_count[intra]++;
3482
3483     for(i=0; i<64; i++){
3484         int level= block[i];
3485
3486         if(level){
3487             if(level>0){
3488                 s->dct_error_sum[intra][i] += level;
3489                 level -= s->dct_offset[intra][i];
3490                 if(level<0) level=0;
3491             }else{
3492                 s->dct_error_sum[intra][i] -= level;
3493                 level += s->dct_offset[intra][i];
3494                 if(level>0) level=0;
3495             }
3496             block[i]= level;
3497         }
3498     }
3499 }
3500
3501 static int dct_quantize_trellis_c(MpegEncContext *s,
3502                                   int16_t *block, int n,
3503                                   int qscale, int *overflow){
3504     const int *qmat;
3505     const uint8_t *scantable= s->intra_scantable.scantable;
3506     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3507     int max=0;
3508     unsigned int threshold1, threshold2;
3509     int bias=0;
3510     int run_tab[65];
3511     int level_tab[65];
3512     int score_tab[65];
3513     int survivor[65];
3514     int survivor_count;
3515     int last_run=0;
3516     int last_level=0;
3517     int last_score= 0;
3518     int last_i;
3519     int coeff[2][64];
3520     int coeff_count[64];
3521     int qmul, qadd, start_i, last_non_zero, i, dc;
3522     const int esc_length= s->ac_esc_length;
3523     uint8_t * length;
3524     uint8_t * last_length;
3525     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3526
3527     s->fdsp.fdct(block);
3528
3529     if(s->dct_error_sum)
3530         s->denoise_dct(s, block);
3531     qmul= qscale*16;
3532     qadd= ((qscale-1)|1)*8;
3533
3534     if (s->mb_intra) {
3535         int q;
3536         if (!s->h263_aic) {
3537             if (n < 4)
3538                 q = s->y_dc_scale;
3539             else
3540                 q = s->c_dc_scale;
3541             q = q << 3;
3542         } else{
3543             /* For AIC we skip quant/dequant of INTRADC */
3544             q = 1 << 3;
3545             qadd=0;
3546         }
3547
3548         /* note: block[0] is assumed to be positive */
3549         block[0] = (block[0] + (q >> 1)) / q;
3550         start_i = 1;
3551         last_non_zero = 0;
3552         qmat = s->q_intra_matrix[qscale];
3553         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3554             bias= 1<<(QMAT_SHIFT-1);
3555         length     = s->intra_ac_vlc_length;
3556         last_length= s->intra_ac_vlc_last_length;
3557     } else {
3558         start_i = 0;
3559         last_non_zero = -1;
3560         qmat = s->q_inter_matrix[qscale];
3561         length     = s->inter_ac_vlc_length;
3562         last_length= s->inter_ac_vlc_last_length;
3563     }
3564     last_i= start_i;
3565
3566     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3567     threshold2= (threshold1<<1);
3568
3569     for(i=63; i>=start_i; i--) {
3570         const int j = scantable[i];
3571         int level = block[j] * qmat[j];
3572
3573         if(((unsigned)(level+threshold1))>threshold2){
3574             last_non_zero = i;
3575             break;
3576         }
3577     }
3578
3579     for(i=start_i; i<=last_non_zero; i++) {
3580         const int j = scantable[i];
3581         int level = block[j] * qmat[j];
3582
3583 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3584 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3585         if(((unsigned)(level+threshold1))>threshold2){
3586             if(level>0){
3587                 level= (bias + level)>>QMAT_SHIFT;
3588                 coeff[0][i]= level;
3589                 coeff[1][i]= level-1;
3590 //                coeff[2][k]= level-2;
3591             }else{
3592                 level= (bias - level)>>QMAT_SHIFT;
3593                 coeff[0][i]= -level;
3594                 coeff[1][i]= -level+1;
3595 //                coeff[2][k]= -level+2;
3596             }
3597             coeff_count[i]= FFMIN(level, 2);
3598             assert(coeff_count[i]);
3599             max |=level;
3600         }else{
3601             coeff[0][i]= (level>>31)|1;
3602             coeff_count[i]= 1;
3603         }
3604     }
3605
3606     *overflow= s->max_qcoeff < max; //overflow might have happened
3607
3608     if(last_non_zero < start_i){
3609         memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
3610         return last_non_zero;
3611     }
3612
3613     score_tab[start_i]= 0;
3614     survivor[0]= start_i;
3615     survivor_count= 1;
3616
3617     for(i=start_i; i<=last_non_zero; i++){
3618         int level_index, j, zero_distortion;
3619         int dct_coeff= FFABS(block[ scantable[i] ]);
3620         int best_score=256*256*256*120;
3621
3622         if (s->fdsp.fdct == ff_fdct_ifast)
3623             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3624         zero_distortion= dct_coeff*dct_coeff;
3625
3626         for(level_index=0; level_index < coeff_count[i]; level_index++){
3627             int distortion;
3628             int level= coeff[level_index][i];
3629             const int alevel= FFABS(level);
3630             int unquant_coeff;
3631
3632             assert(level);
3633
3634             if(s->out_format == FMT_H263){
3635                 unquant_coeff= alevel*qmul + qadd;
3636             }else{ //MPEG1
3637                 j = s->idsp.idct_permutation[scantable[i]]; // FIXME: optimize
3638                 if(s->mb_intra){
3639                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3640                         unquant_coeff =   (unquant_coeff - 1) | 1;
3641                 }else{
3642                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3643                         unquant_coeff =   (unquant_coeff - 1) | 1;
3644                 }
3645                 unquant_coeff<<= 3;
3646             }
3647
3648             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3649             level+=64;
3650             if((level&(~127)) == 0){
3651                 for(j=survivor_count-1; j>=0; j--){
3652                     int run= i - survivor[j];
3653                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3654                     score += score_tab[i-run];
3655
3656                     if(score < best_score){
3657                         best_score= score;
3658                         run_tab[i+1]= run;
3659                         level_tab[i+1]= level-64;
3660                     }
3661                 }
3662
3663                 if(s->out_format == FMT_H263){
3664                     for(j=survivor_count-1; j>=0; j--){
3665                         int run= i - survivor[j];
3666                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3667                         score += score_tab[i-run];
3668                         if(score < last_score){
3669                             last_score= score;
3670                             last_run= run;
3671                             last_level= level-64;
3672                             last_i= i+1;
3673                         }
3674                     }
3675                 }
3676             }else{
3677                 distortion += esc_length*lambda;
3678                 for(j=survivor_count-1; j>=0; j--){
3679                     int run= i - survivor[j];
3680                     int score= distortion + score_tab[i-run];
3681
3682                     if(score < best_score){
3683                         best_score= score;
3684                         run_tab[i+1]= run;
3685                         level_tab[i+1]= level-64;
3686                     }
3687                 }
3688
3689                 if(s->out_format == FMT_H263){
3690                   for(j=survivor_count-1; j>=0; j--){
3691                         int run= i - survivor[j];
3692                         int score= distortion + score_tab[i-run];
3693                         if(score < last_score){
3694                             last_score= score;
3695                             last_run= run;
3696                             last_level= level-64;
3697                             last_i= i+1;
3698                         }
3699                     }
3700                 }
3701             }
3702         }
3703
3704         score_tab[i+1]= best_score;
3705
3706         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
3707         if(last_non_zero <= 27){
3708             for(; survivor_count; survivor_count--){
3709                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3710                     break;
3711             }
3712         }else{
3713             for(; survivor_count; survivor_count--){
3714                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3715                     break;
3716             }
3717         }
3718
3719         survivor[ survivor_count++ ]= i+1;
3720     }
3721
3722     if(s->out_format != FMT_H263){
3723         last_score= 256*256*256*120;
3724         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3725             int score= score_tab[i];
3726             if(i) score += lambda*2; //FIXME exacter?
3727
3728             if(score < last_score){
3729                 last_score= score;
3730                 last_i= i;
3731                 last_level= level_tab[i];
3732                 last_run= run_tab[i];
3733             }
3734         }
3735     }
3736
3737     s->coded_score[n] = last_score;
3738
3739     dc= FFABS(block[0]);
3740     last_non_zero= last_i - 1;
3741     memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
3742
3743     if(last_non_zero < start_i)
3744         return last_non_zero;
3745
3746     if(last_non_zero == 0 && start_i == 0){
3747         int best_level= 0;
3748         int best_score= dc * dc;
3749
3750         for(i=0; i<coeff_count[0]; i++){
3751             int level= coeff[i][0];
3752             int alevel= FFABS(level);
3753             int unquant_coeff, score, distortion;
3754
3755             if(s->out_format == FMT_H263){
3756                     unquant_coeff= (alevel*qmul + qadd)>>3;
3757             }else{ //MPEG1
3758                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3759                     unquant_coeff =   (unquant_coeff - 1) | 1;
3760             }
3761             unquant_coeff = (unquant_coeff + 4) >> 3;
3762             unquant_coeff<<= 3 + 3;
3763
3764             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3765             level+=64;
3766             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3767             else                    score= distortion + esc_length*lambda;
3768
3769             if(score < best_score){
3770                 best_score= score;
3771                 best_level= level - 64;
3772             }
3773         }
3774         block[0]= best_level;
3775         s->coded_score[n] = best_score - dc*dc;
3776         if(best_level == 0) return -1;
3777         else                return last_non_zero;
3778     }
3779
3780     i= last_i;
3781     assert(last_level);
3782
3783     block[ perm_scantable[last_non_zero] ]= last_level;
3784     i -= last_run + 1;
3785
3786     for(; i>start_i; i -= run_tab[i] + 1){
3787         block[ perm_scantable[i-1] ]= level_tab[i];
3788     }
3789
3790     return last_non_zero;
3791 }
3792
3793 //#define REFINE_STATS 1
3794 static int16_t basis[64][64];
3795
3796 static void build_basis(uint8_t *perm){
3797     int i, j, x, y;
3798     emms_c();
3799     for(i=0; i<8; i++){
3800         for(j=0; j<8; j++){
3801             for(y=0; y<8; y++){
3802                 for(x=0; x<8; x++){
3803                     double s= 0.25*(1<<BASIS_SHIFT);
3804                     int index= 8*i + j;
3805                     int perm_index= perm[index];
3806                     if(i==0) s*= sqrt(0.5);
3807                     if(j==0) s*= sqrt(0.5);
3808                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3809                 }
3810             }
3811         }
3812     }
3813 }
3814
3815 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3816                         int16_t *block, int16_t *weight, int16_t *orig,
3817                         int n, int qscale){
3818     int16_t rem[64];
3819     LOCAL_ALIGNED_16(int16_t, d1, [64]);
3820     const uint8_t *scantable= s->intra_scantable.scantable;
3821     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3822 //    unsigned int threshold1, threshold2;
3823 //    int bias=0;
3824     int run_tab[65];
3825     int prev_run=0;
3826     int prev_level=0;
3827     int qmul, qadd, start_i, last_non_zero, i, dc;
3828     uint8_t * length;
3829     uint8_t * last_length;
3830     int lambda;
3831     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3832 #ifdef REFINE_STATS
3833 static int count=0;
3834 static int after_last=0;
3835 static int to_zero=0;
3836 static int from_zero=0;
3837 static int raise=0;
3838 static int lower=0;
3839 static int messed_sign=0;
3840 #endif
3841
3842     if(basis[0][0] == 0)
3843         build_basis(s->idsp.idct_permutation);
3844
3845     qmul= qscale*2;
3846     qadd= (qscale-1)|1;
3847     if (s->mb_intra) {
3848         if (!s->h263_aic) {
3849             if (n < 4)
3850                 q = s->y_dc_scale;
3851             else
3852                 q = s->c_dc_scale;
3853         } else{
3854             /* For AIC we skip quant/dequant of INTRADC */
3855             q = 1;
3856             qadd=0;
3857         }
3858         q <<= RECON_SHIFT-3;
3859         /* note: block[0] is assumed to be positive */
3860         dc= block[0]*q;
3861 //        block[0] = (block[0] + (q >> 1)) / q;
3862         start_i = 1;
3863 //        if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3864 //            bias= 1<<(QMAT_SHIFT-1);
3865         length     = s->intra_ac_vlc_length;
3866         last_length= s->intra_ac_vlc_last_length;
3867     } else {
3868         dc= 0;
3869         start_i = 0;
3870         length     = s->inter_ac_vlc_length;
3871         last_length= s->inter_ac