mpegvideo: rv10: Move function declaration to a separate header
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of Libav.
9  *
10  * Libav is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * Libav is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with Libav; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include <stdint.h>
31
32 #include "libavutil/internal.h"
33 #include "libavutil/intmath.h"
34 #include "libavutil/mathematics.h"
35 #include "libavutil/pixdesc.h"
36 #include "libavutil/opt.h"
37 #include "libavutil/timer.h"
38 #include "avcodec.h"
39 #include "dct.h"
40 #include "idctdsp.h"
41 #include "mpeg12.h"
42 #include "mpegvideo.h"
43 #include "mpegvideodata.h"
44 #include "h261.h"
45 #include "h263.h"
46 #include "h263data.h"
47 #include "mjpegenc_common.h"
48 #include "mathops.h"
49 #include "mpegutils.h"
50 #include "mjpegenc.h"
51 #include "msmpeg4.h"
52 #include "pixblockdsp.h"
53 #include "qpeldsp.h"
54 #include "faandct.h"
55 #include "thread.h"
56 #include "aandcttab.h"
57 #include "flv.h"
58 #include "mpeg4video.h"
59 #include "internal.h"
60 #include "bytestream.h"
61 #include "wmv2.h"
62 #include "rv10.h"
63 #include <limits.h>
64
65 #define QUANT_BIAS_SHIFT 8
66
67 #define QMAT_SHIFT_MMX 16
68 #define QMAT_SHIFT 22
69
70 static int encode_picture(MpegEncContext *s, int picture_number);
71 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
72 static int sse_mb(MpegEncContext *s);
73 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
74 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
75
76 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
77 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
78
79 const AVOption ff_mpv_generic_options[] = {
80     FF_MPV_COMMON_OPTS
81     { NULL },
82 };
83
84 void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64],
85                        uint16_t (*qmat16)[2][64],
86                        const uint16_t *quant_matrix,
87                        int bias, int qmin, int qmax, int intra)
88 {
89     FDCTDSPContext *fdsp = &s->fdsp;
90     int qscale;
91     int shift = 0;
92
93     for (qscale = qmin; qscale <= qmax; qscale++) {
94         int i;
95         if (fdsp->fdct == ff_jpeg_fdct_islow_8  ||
96 #if CONFIG_FAANDCT
97             fdsp->fdct == ff_faandct            ||
98 #endif /* CONFIG_FAANDCT */
99             fdsp->fdct == ff_jpeg_fdct_islow_10) {
100             for (i = 0; i < 64; i++) {
101                 const int j = s->idsp.idct_permutation[i];
102                 int64_t den = (int64_t) qscale * quant_matrix[j];
103                 /* 16 <= qscale * quant_matrix[i] <= 7905
104                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
105                  *             19952 <=              x  <= 249205026
106                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
107                  *           3444240 >= (1 << 36) / (x) >= 275 */
108
109                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
110             }
111         } else if (fdsp->fdct == ff_fdct_ifast) {
112             for (i = 0; i < 64; i++) {
113                 const int j = s->idsp.idct_permutation[i];
114                 int64_t den = ff_aanscales[i] * (int64_t) qscale * quant_matrix[j];
115                 /* 16 <= qscale * quant_matrix[i] <= 7905
116                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
117                  *             19952 <=              x  <= 249205026
118                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
119                  *           3444240 >= (1 << 36) / (x) >= 275 */
120
121                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / den);
122             }
123         } else {
124             for (i = 0; i < 64; i++) {
125                 const int j = s->idsp.idct_permutation[i];
126                 int64_t den = (int64_t) qscale * quant_matrix[j];
127                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
128                  * Assume x = qscale * quant_matrix[i]
129                  * So             16 <=              x  <= 7905
130                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
131                  * so          32768 >= (1 << 19) / (x) >= 67 */
132                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
133                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
134                 //                    (qscale * quant_matrix[i]);
135                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / den;
136
137                 if (qmat16[qscale][0][i] == 0 ||
138                     qmat16[qscale][0][i] == 128 * 256)
139                     qmat16[qscale][0][i] = 128 * 256 - 1;
140                 qmat16[qscale][1][i] =
141                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
142                                 qmat16[qscale][0][i]);
143             }
144         }
145
146         for (i = intra; i < 64; i++) {
147             int64_t max = 8191;
148             if (fdsp->fdct == ff_fdct_ifast) {
149                 max = (8191LL * ff_aanscales[i]) >> 14;
150             }
151             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
152                 shift++;
153             }
154         }
155     }
156     if (shift) {
157         av_log(NULL, AV_LOG_INFO,
158                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
159                QMAT_SHIFT - shift);
160     }
161 }
162
163 static inline void update_qscale(MpegEncContext *s)
164 {
165     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
166                 (FF_LAMBDA_SHIFT + 7);
167     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
168
169     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
170                  FF_LAMBDA_SHIFT;
171 }
172
173 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
174 {
175     int i;
176
177     if (matrix) {
178         put_bits(pb, 1, 1);
179         for (i = 0; i < 64; i++) {
180             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
181         }
182     } else
183         put_bits(pb, 1, 0);
184 }
185
186 /**
187  * init s->current_picture.qscale_table from s->lambda_table
188  */
189 void ff_init_qscale_tab(MpegEncContext *s)
190 {
191     int8_t * const qscale_table = s->current_picture.qscale_table;
192     int i;
193
194     for (i = 0; i < s->mb_num; i++) {
195         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
196         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
197         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
198                                                   s->avctx->qmax);
199     }
200 }
201
202 static void update_duplicate_context_after_me(MpegEncContext *dst,
203                                               MpegEncContext *src)
204 {
205 #define COPY(a) dst->a= src->a
206     COPY(pict_type);
207     COPY(current_picture);
208     COPY(f_code);
209     COPY(b_code);
210     COPY(qscale);
211     COPY(lambda);
212     COPY(lambda2);
213     COPY(picture_in_gop_number);
214     COPY(gop_picture_number);
215     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
216     COPY(progressive_frame);    // FIXME don't set in encode_header
217     COPY(partitioned_frame);    // FIXME don't set in encode_header
218 #undef COPY
219 }
220
221 /**
222  * Set the given MpegEncContext to defaults for encoding.
223  * the changed fields will not depend upon the prior state of the MpegEncContext.
224  */
225 static void mpv_encode_defaults(MpegEncContext *s)
226 {
227     int i;
228     ff_mpv_common_defaults(s);
229
230     for (i = -16; i < 16; i++) {
231         default_fcode_tab[i + MAX_MV] = 1;
232     }
233     s->me.mv_penalty = default_mv_penalty;
234     s->fcode_tab     = default_fcode_tab;
235
236     s->input_picture_number  = 0;
237     s->picture_in_gop_number = 0;
238 }
239
240 /* init video encoder */
241 av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
242 {
243     MpegEncContext *s = avctx->priv_data;
244     int i, ret, format_supported;
245
246     mpv_encode_defaults(s);
247
248     switch (avctx->codec_id) {
249     case AV_CODEC_ID_MPEG2VIDEO:
250         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
251             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
252             av_log(avctx, AV_LOG_ERROR,
253                    "only YUV420 and YUV422 are supported\n");
254             return -1;
255         }
256         break;
257     case AV_CODEC_ID_MJPEG:
258         format_supported = 0;
259         /* JPEG color space */
260         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
261             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
262             (avctx->color_range == AVCOL_RANGE_JPEG &&
263              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
264               avctx->pix_fmt == AV_PIX_FMT_YUV422P)))
265             format_supported = 1;
266         /* MPEG color space */
267         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
268                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
269                   avctx->pix_fmt == AV_PIX_FMT_YUV422P))
270             format_supported = 1;
271
272         if (!format_supported) {
273             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
274             return -1;
275         }
276         break;
277     default:
278         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
279             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
280             return -1;
281         }
282     }
283
284     switch (avctx->pix_fmt) {
285     case AV_PIX_FMT_YUVJ422P:
286     case AV_PIX_FMT_YUV422P:
287         s->chroma_format = CHROMA_422;
288         break;
289     case AV_PIX_FMT_YUVJ420P:
290     case AV_PIX_FMT_YUV420P:
291     default:
292         s->chroma_format = CHROMA_420;
293         break;
294     }
295
296     s->bit_rate = avctx->bit_rate;
297     s->width    = avctx->width;
298     s->height   = avctx->height;
299     if (avctx->gop_size > 600 &&
300         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
301         av_log(avctx, AV_LOG_ERROR,
302                "Warning keyframe interval too large! reducing it ...\n");
303         avctx->gop_size = 600;
304     }
305     s->gop_size     = avctx->gop_size;
306     s->avctx        = avctx;
307     if (avctx->max_b_frames > MAX_B_FRAMES) {
308         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
309                "is %d.\n", MAX_B_FRAMES);
310     }
311     s->max_b_frames = avctx->max_b_frames;
312     s->codec_id     = avctx->codec->id;
313     s->strict_std_compliance = avctx->strict_std_compliance;
314     s->quarter_sample     = (avctx->flags & CODEC_FLAG_QPEL) != 0;
315     s->mpeg_quant         = avctx->mpeg_quant;
316     s->rtp_mode           = !!avctx->rtp_payload_size;
317     s->intra_dc_precision = avctx->intra_dc_precision;
318     s->user_specified_pts = AV_NOPTS_VALUE;
319
320     if (s->gop_size <= 1) {
321         s->intra_only = 1;
322         s->gop_size   = 12;
323     } else {
324         s->intra_only = 0;
325     }
326
327     s->me_method = avctx->me_method;
328
329     /* Fixed QSCALE */
330     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
331
332 #if FF_API_MPV_OPT
333     FF_DISABLE_DEPRECATION_WARNINGS
334     if (avctx->border_masking != 0.0)
335         s->border_masking = avctx->border_masking;
336     FF_ENABLE_DEPRECATION_WARNINGS
337 #endif
338
339     s->adaptive_quant = (s->avctx->lumi_masking ||
340                          s->avctx->dark_masking ||
341                          s->avctx->temporal_cplx_masking ||
342                          s->avctx->spatial_cplx_masking  ||
343                          s->avctx->p_masking      ||
344                          s->border_masking ||
345                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
346                         !s->fixed_qscale;
347
348     s->loop_filter = !!(s->avctx->flags & CODEC_FLAG_LOOP_FILTER);
349
350     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
351         av_log(avctx, AV_LOG_ERROR,
352                "a vbv buffer size is needed, "
353                "for encoding with a maximum bitrate\n");
354         return -1;
355     }
356
357     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
358         av_log(avctx, AV_LOG_INFO,
359                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
360     }
361
362     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
363         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
364         return -1;
365     }
366
367     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
368         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
369         return -1;
370     }
371
372     if (avctx->rc_max_rate &&
373         avctx->rc_max_rate == avctx->bit_rate &&
374         avctx->rc_max_rate != avctx->rc_min_rate) {
375         av_log(avctx, AV_LOG_INFO,
376                "impossible bitrate constraints, this will fail\n");
377     }
378
379     if (avctx->rc_buffer_size &&
380         avctx->bit_rate * (int64_t)avctx->time_base.num >
381             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
382         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
383         return -1;
384     }
385
386     if (!s->fixed_qscale &&
387         avctx->bit_rate * av_q2d(avctx->time_base) >
388             avctx->bit_rate_tolerance) {
389         av_log(avctx, AV_LOG_ERROR,
390                "bitrate tolerance too small for bitrate\n");
391         return -1;
392     }
393
394     if (s->avctx->rc_max_rate &&
395         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
396         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
397          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
398         90000LL * (avctx->rc_buffer_size - 1) >
399             s->avctx->rc_max_rate * 0xFFFFLL) {
400         av_log(avctx, AV_LOG_INFO,
401                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
402                "specified vbv buffer is too large for the given bitrate!\n");
403     }
404
405     if ((s->avctx->flags & CODEC_FLAG_4MV) && s->codec_id != AV_CODEC_ID_MPEG4 &&
406         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
407         s->codec_id != AV_CODEC_ID_FLV1) {
408         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
409         return -1;
410     }
411
412     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
413         av_log(avctx, AV_LOG_ERROR,
414                "OBMC is only supported with simple mb decision\n");
415         return -1;
416     }
417
418     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
419         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
420         return -1;
421     }
422
423     if (s->max_b_frames                    &&
424         s->codec_id != AV_CODEC_ID_MPEG4      &&
425         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
426         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
427         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
428         return -1;
429     }
430
431     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
432          s->codec_id == AV_CODEC_ID_H263  ||
433          s->codec_id == AV_CODEC_ID_H263P) &&
434         (avctx->sample_aspect_ratio.num > 255 ||
435          avctx->sample_aspect_ratio.den > 255)) {
436         av_log(avctx, AV_LOG_ERROR,
437                "Invalid pixel aspect ratio %i/%i, limit is 255/255\n",
438                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
439         return -1;
440     }
441
442     if ((s->avctx->flags & (CODEC_FLAG_INTERLACED_DCT | CODEC_FLAG_INTERLACED_ME)) &&
443         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
444         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
445         return -1;
446     }
447
448     // FIXME mpeg2 uses that too
449     if (s->mpeg_quant && s->codec_id != AV_CODEC_ID_MPEG4) {
450         av_log(avctx, AV_LOG_ERROR,
451                "mpeg2 style quantization not supported by codec\n");
452         return -1;
453     }
454
455     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
456         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
457         return -1;
458     }
459
460     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
461         s->avctx->mb_decision != FF_MB_DECISION_RD) {
462         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
463         return -1;
464     }
465
466     if (s->avctx->scenechange_threshold < 1000000000 &&
467         (s->avctx->flags & CODEC_FLAG_CLOSED_GOP)) {
468         av_log(avctx, AV_LOG_ERROR,
469                "closed gop with scene change detection are not supported yet, "
470                "set threshold to 1000000000\n");
471         return -1;
472     }
473
474     if (s->avctx->flags & CODEC_FLAG_LOW_DELAY) {
475         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
476             av_log(avctx, AV_LOG_ERROR,
477                   "low delay forcing is only available for mpeg2\n");
478             return -1;
479         }
480         if (s->max_b_frames != 0) {
481             av_log(avctx, AV_LOG_ERROR,
482                    "b frames cannot be used with low delay\n");
483             return -1;
484         }
485     }
486
487     if (s->q_scale_type == 1) {
488         if (avctx->qmax > 12) {
489             av_log(avctx, AV_LOG_ERROR,
490                    "non linear quant only supports qmax <= 12 currently\n");
491             return -1;
492         }
493     }
494
495     if (s->avctx->thread_count > 1         &&
496         s->codec_id != AV_CODEC_ID_MPEG4      &&
497         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
498         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
499         (s->codec_id != AV_CODEC_ID_H263P)) {
500         av_log(avctx, AV_LOG_ERROR,
501                "multi threaded encoding not supported by codec\n");
502         return -1;
503     }
504
505     if (s->avctx->thread_count < 1) {
506         av_log(avctx, AV_LOG_ERROR,
507                "automatic thread number detection not supported by codec,"
508                "patch welcome\n");
509         return -1;
510     }
511
512     if (s->avctx->thread_count > 1)
513         s->rtp_mode = 1;
514
515     if (!avctx->time_base.den || !avctx->time_base.num) {
516         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
517         return -1;
518     }
519
520     if (avctx->b_frame_strategy && (avctx->flags & CODEC_FLAG_PASS2)) {
521         av_log(avctx, AV_LOG_INFO,
522                "notice: b_frame_strategy only affects the first pass\n");
523         avctx->b_frame_strategy = 0;
524     }
525
526     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
527     if (i > 1) {
528         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
529         avctx->time_base.den /= i;
530         avctx->time_base.num /= i;
531         //return -1;
532     }
533
534     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
535         s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG) {
536         // (a + x * 3 / 8) / x
537         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
538         s->inter_quant_bias = 0;
539     } else {
540         s->intra_quant_bias = 0;
541         // (a - x / 4) / x
542         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
543     }
544
545     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
546         s->intra_quant_bias = avctx->intra_quant_bias;
547     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
548         s->inter_quant_bias = avctx->inter_quant_bias;
549
550     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
551         s->avctx->time_base.den > (1 << 16) - 1) {
552         av_log(avctx, AV_LOG_ERROR,
553                "timebase %d/%d not supported by MPEG 4 standard, "
554                "the maximum admitted value for the timebase denominator "
555                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
556                (1 << 16) - 1);
557         return -1;
558     }
559     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
560
561     switch (avctx->codec->id) {
562     case AV_CODEC_ID_MPEG1VIDEO:
563         s->out_format = FMT_MPEG1;
564         s->low_delay  = !!(s->avctx->flags & CODEC_FLAG_LOW_DELAY);
565         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
566         break;
567     case AV_CODEC_ID_MPEG2VIDEO:
568         s->out_format = FMT_MPEG1;
569         s->low_delay  = !!(s->avctx->flags & CODEC_FLAG_LOW_DELAY);
570         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
571         s->rtp_mode   = 1;
572         break;
573     case AV_CODEC_ID_MJPEG:
574         s->out_format = FMT_MJPEG;
575         s->intra_only = 1; /* force intra only for jpeg */
576         if (!CONFIG_MJPEG_ENCODER ||
577             ff_mjpeg_encode_init(s) < 0)
578             return -1;
579         avctx->delay = 0;
580         s->low_delay = 1;
581         break;
582     case AV_CODEC_ID_H261:
583         if (!CONFIG_H261_ENCODER)
584             return -1;
585         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
586             av_log(avctx, AV_LOG_ERROR,
587                    "The specified picture size of %dx%d is not valid for the "
588                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
589                     s->width, s->height);
590             return -1;
591         }
592         s->out_format = FMT_H261;
593         avctx->delay  = 0;
594         s->low_delay  = 1;
595         s->rtp_mode   = 0; /* Sliced encoding not supported */
596         break;
597     case AV_CODEC_ID_H263:
598         if (!CONFIG_H263_ENCODER)
599         return -1;
600         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
601                              s->width, s->height) == 8) {
602             av_log(avctx, AV_LOG_INFO,
603                    "The specified picture size of %dx%d is not valid for "
604                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
605                    "352x288, 704x576, and 1408x1152."
606                    "Try H.263+.\n", s->width, s->height);
607             return -1;
608         }
609         s->out_format = FMT_H263;
610         avctx->delay  = 0;
611         s->low_delay  = 1;
612         break;
613     case AV_CODEC_ID_H263P:
614         s->out_format = FMT_H263;
615         s->h263_plus  = 1;
616         /* Fx */
617         s->h263_aic        = (avctx->flags & CODEC_FLAG_AC_PRED) ? 1 : 0;
618         s->modified_quant  = s->h263_aic;
619         s->loop_filter     = (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
620         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
621
622         /* /Fx */
623         /* These are just to be sure */
624         avctx->delay = 0;
625         s->low_delay = 1;
626         break;
627     case AV_CODEC_ID_FLV1:
628         s->out_format      = FMT_H263;
629         s->h263_flv        = 2; /* format = 1; 11-bit codes */
630         s->unrestricted_mv = 1;
631         s->rtp_mode  = 0; /* don't allow GOB */
632         avctx->delay = 0;
633         s->low_delay = 1;
634         break;
635     case AV_CODEC_ID_RV10:
636         s->out_format = FMT_H263;
637         avctx->delay  = 0;
638         s->low_delay  = 1;
639         break;
640     case AV_CODEC_ID_RV20:
641         s->out_format      = FMT_H263;
642         avctx->delay       = 0;
643         s->low_delay       = 1;
644         s->modified_quant  = 1;
645         s->h263_aic        = 1;
646         s->h263_plus       = 1;
647         s->loop_filter     = 1;
648         s->unrestricted_mv = 0;
649         break;
650     case AV_CODEC_ID_MPEG4:
651         s->out_format      = FMT_H263;
652         s->h263_pred       = 1;
653         s->unrestricted_mv = 1;
654         s->low_delay       = s->max_b_frames ? 0 : 1;
655         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
656         break;
657     case AV_CODEC_ID_MSMPEG4V2:
658         s->out_format      = FMT_H263;
659         s->h263_pred       = 1;
660         s->unrestricted_mv = 1;
661         s->msmpeg4_version = 2;
662         avctx->delay       = 0;
663         s->low_delay       = 1;
664         break;
665     case AV_CODEC_ID_MSMPEG4V3:
666         s->out_format        = FMT_H263;
667         s->h263_pred         = 1;
668         s->unrestricted_mv   = 1;
669         s->msmpeg4_version   = 3;
670         s->flipflop_rounding = 1;
671         avctx->delay         = 0;
672         s->low_delay         = 1;
673         break;
674     case AV_CODEC_ID_WMV1:
675         s->out_format        = FMT_H263;
676         s->h263_pred         = 1;
677         s->unrestricted_mv   = 1;
678         s->msmpeg4_version   = 4;
679         s->flipflop_rounding = 1;
680         avctx->delay         = 0;
681         s->low_delay         = 1;
682         break;
683     case AV_CODEC_ID_WMV2:
684         s->out_format        = FMT_H263;
685         s->h263_pred         = 1;
686         s->unrestricted_mv   = 1;
687         s->msmpeg4_version   = 5;
688         s->flipflop_rounding = 1;
689         avctx->delay         = 0;
690         s->low_delay         = 1;
691         break;
692     default:
693         return -1;
694     }
695
696     avctx->has_b_frames = !s->low_delay;
697
698     s->encoding = 1;
699
700     s->progressive_frame    =
701     s->progressive_sequence = !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT |
702                                                 CODEC_FLAG_INTERLACED_ME) ||
703                                 s->alternate_scan);
704
705     /* init */
706     ff_mpv_idct_init(s);
707     if (ff_mpv_common_init(s) < 0)
708         return -1;
709
710     if (ARCH_X86)
711         ff_mpv_encode_init_x86(s);
712
713     ff_fdctdsp_init(&s->fdsp, avctx);
714     ff_me_cmp_init(&s->mecc, avctx);
715     ff_mpegvideoencdsp_init(&s->mpvencdsp, avctx);
716     ff_pixblockdsp_init(&s->pdsp, avctx);
717     ff_qpeldsp_init(&s->qdsp);
718
719     s->avctx->coded_frame = s->current_picture.f;
720
721     if (s->msmpeg4_version) {
722         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
723                           2 * 2 * (MAX_LEVEL + 1) *
724                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
725     }
726     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
727
728     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
729     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
730     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
731     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
732     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
733                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
734     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
735                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
736
737     if (s->avctx->noise_reduction) {
738         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
739                           2 * 64 * sizeof(uint16_t), fail);
740     }
741
742     if (CONFIG_H263_ENCODER)
743         ff_h263dsp_init(&s->h263dsp);
744     if (!s->dct_quantize)
745         s->dct_quantize = ff_dct_quantize_c;
746     if (!s->denoise_dct)
747         s->denoise_dct  = denoise_dct_c;
748     s->fast_dct_quantize = s->dct_quantize;
749     if (avctx->trellis)
750         s->dct_quantize  = dct_quantize_trellis_c;
751
752     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
753         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
754
755     s->quant_precision = 5;
756
757     ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      s->avctx->ildct_cmp);
758     ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->avctx->frame_skip_cmp);
759
760     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
761         ff_h261_encode_init(s);
762     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
763         ff_h263_encode_init(s);
764     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
765         if ((ret = ff_msmpeg4_encode_init(s)) < 0)
766             return ret;
767     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
768         && s->out_format == FMT_MPEG1)
769         ff_mpeg1_encode_init(s);
770
771     /* init q matrix */
772     for (i = 0; i < 64; i++) {
773         int j = s->idsp.idct_permutation[i];
774         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
775             s->mpeg_quant) {
776             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
777             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
778         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
779             s->intra_matrix[j] =
780             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
781         } else {
782             /* mpeg1/2 */
783             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
784             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
785         }
786         if (s->avctx->intra_matrix)
787             s->intra_matrix[j] = s->avctx->intra_matrix[i];
788         if (s->avctx->inter_matrix)
789             s->inter_matrix[j] = s->avctx->inter_matrix[i];
790     }
791
792     /* precompute matrix */
793     /* for mjpeg, we do include qscale in the matrix */
794     if (s->out_format != FMT_MJPEG) {
795         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
796                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
797                           31, 1);
798         ff_convert_matrix(s, s->q_inter_matrix, s->q_inter_matrix16,
799                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
800                           31, 0);
801     }
802
803     if (ff_rate_control_init(s) < 0)
804         return -1;
805
806 #if FF_API_ERROR_RATE
807     FF_DISABLE_DEPRECATION_WARNINGS
808     if (avctx->error_rate)
809         s->error_rate = avctx->error_rate;
810     FF_ENABLE_DEPRECATION_WARNINGS;
811 #endif
812
813 #if FF_API_NORMALIZE_AQP
814     FF_DISABLE_DEPRECATION_WARNINGS
815     if (avctx->flags & CODEC_FLAG_NORMALIZE_AQP)
816         s->mpv_flags |= FF_MPV_FLAG_NAQ;
817     FF_ENABLE_DEPRECATION_WARNINGS;
818 #endif
819
820 #if FF_API_MV0
821     FF_DISABLE_DEPRECATION_WARNINGS
822     if (avctx->flags & CODEC_FLAG_MV0)
823         s->mpv_flags |= FF_MPV_FLAG_MV0;
824     FF_ENABLE_DEPRECATION_WARNINGS
825 #endif
826
827 #if FF_API_MPV_OPT
828     FF_DISABLE_DEPRECATION_WARNINGS
829     if (avctx->rc_qsquish != 0.0)
830         s->rc_qsquish = avctx->rc_qsquish;
831     if (avctx->rc_qmod_amp != 0.0)
832         s->rc_qmod_amp = avctx->rc_qmod_amp;
833     if (avctx->rc_qmod_freq)
834         s->rc_qmod_freq = avctx->rc_qmod_freq;
835     if (avctx->rc_buffer_aggressivity != 1.0)
836         s->rc_buffer_aggressivity = avctx->rc_buffer_aggressivity;
837     if (avctx->rc_initial_cplx != 0.0)
838         s->rc_initial_cplx = avctx->rc_initial_cplx;
839     if (avctx->lmin)
840         s->lmin = avctx->lmin;
841     if (avctx->lmax)
842         s->lmax = avctx->lmax;
843
844     if (avctx->rc_eq) {
845         av_freep(&s->rc_eq);
846         s->rc_eq = av_strdup(avctx->rc_eq);
847         if (!s->rc_eq)
848             return AVERROR(ENOMEM);
849     }
850     FF_ENABLE_DEPRECATION_WARNINGS
851 #endif
852
853     if (avctx->b_frame_strategy == 2) {
854         for (i = 0; i < s->max_b_frames + 2; i++) {
855             s->tmp_frames[i] = av_frame_alloc();
856             if (!s->tmp_frames[i])
857                 return AVERROR(ENOMEM);
858
859             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
860             s->tmp_frames[i]->width  = s->width  >> avctx->brd_scale;
861             s->tmp_frames[i]->height = s->height >> avctx->brd_scale;
862
863             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
864             if (ret < 0)
865                 return ret;
866         }
867     }
868
869     return 0;
870 fail:
871     ff_mpv_encode_end(avctx);
872     return AVERROR_UNKNOWN;
873 }
874
875 av_cold int ff_mpv_encode_end(AVCodecContext *avctx)
876 {
877     MpegEncContext *s = avctx->priv_data;
878     int i;
879
880     ff_rate_control_uninit(s);
881
882     ff_mpv_common_end(s);
883     if (CONFIG_MJPEG_ENCODER &&
884         s->out_format == FMT_MJPEG)
885         ff_mjpeg_encode_close(s);
886
887     av_freep(&avctx->extradata);
888
889     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
890         av_frame_free(&s->tmp_frames[i]);
891
892     ff_free_picture_tables(&s->new_picture);
893     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
894
895     av_freep(&s->avctx->stats_out);
896     av_freep(&s->ac_stats);
897
898     av_freep(&s->q_intra_matrix);
899     av_freep(&s->q_inter_matrix);
900     av_freep(&s->q_intra_matrix16);
901     av_freep(&s->q_inter_matrix16);
902     av_freep(&s->input_picture);
903     av_freep(&s->reordered_input_picture);
904     av_freep(&s->dct_offset);
905
906     return 0;
907 }
908
909 static int get_sae(uint8_t *src, int ref, int stride)
910 {
911     int x,y;
912     int acc = 0;
913
914     for (y = 0; y < 16; y++) {
915         for (x = 0; x < 16; x++) {
916             acc += FFABS(src[x + y * stride] - ref);
917         }
918     }
919
920     return acc;
921 }
922
923 static int get_intra_count(MpegEncContext *s, uint8_t *src,
924                            uint8_t *ref, int stride)
925 {
926     int x, y, w, h;
927     int acc = 0;
928
929     w = s->width  & ~15;
930     h = s->height & ~15;
931
932     for (y = 0; y < h; y += 16) {
933         for (x = 0; x < w; x += 16) {
934             int offset = x + y * stride;
935             int sad  = s->mecc.sad[0](NULL, src + offset, ref + offset,
936                                       stride, 16);
937             int mean = (s->mpvencdsp.pix_sum(src + offset, stride) + 128) >> 8;
938             int sae  = get_sae(src + offset, mean, stride);
939
940             acc += sae + 500 < sad;
941         }
942     }
943     return acc;
944 }
945
946 static int alloc_picture(MpegEncContext *s, Picture *pic, int shared)
947 {
948     return ff_alloc_picture(s->avctx, pic, &s->me, &s->sc, shared, 1,
949                             s->chroma_x_shift, s->chroma_y_shift, s->out_format,
950                             s->mb_stride, s->mb_height, s->b8_stride,
951                             &s->linesize, &s->uvlinesize);
952 }
953
954 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
955 {
956     Picture *pic = NULL;
957     int64_t pts;
958     int i, display_picture_number = 0, ret;
959     const int encoding_delay = s->max_b_frames ? s->max_b_frames :
960                                                  (s->low_delay ? 0 : 1);
961     int direct = 1;
962
963     if (pic_arg) {
964         pts = pic_arg->pts;
965         display_picture_number = s->input_picture_number++;
966
967         if (pts != AV_NOPTS_VALUE) {
968             if (s->user_specified_pts != AV_NOPTS_VALUE) {
969                 int64_t time = pts;
970                 int64_t last = s->user_specified_pts;
971
972                 if (time <= last) {
973                     av_log(s->avctx, AV_LOG_ERROR,
974                            "Error, Invalid timestamp=%"PRId64", "
975                            "last=%"PRId64"\n", pts, s->user_specified_pts);
976                     return -1;
977                 }
978
979                 if (!s->low_delay && display_picture_number == 1)
980                     s->dts_delta = time - last;
981             }
982             s->user_specified_pts = pts;
983         } else {
984             if (s->user_specified_pts != AV_NOPTS_VALUE) {
985                 s->user_specified_pts =
986                 pts = s->user_specified_pts + 1;
987                 av_log(s->avctx, AV_LOG_INFO,
988                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
989                        pts);
990             } else {
991                 pts = display_picture_number;
992             }
993         }
994     }
995
996     if (pic_arg) {
997         if (!pic_arg->buf[0] ||
998             pic_arg->linesize[0] != s->linesize ||
999             pic_arg->linesize[1] != s->uvlinesize ||
1000             pic_arg->linesize[2] != s->uvlinesize)
1001             direct = 0;
1002         if ((s->width & 15) || (s->height & 15))
1003             direct = 0;
1004
1005         ff_dlog(s->avctx, "%d %d %td %td\n", pic_arg->linesize[0],
1006                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
1007
1008         i = ff_find_unused_picture(s->avctx, s->picture, direct);
1009         if (i < 0)
1010             return i;
1011
1012         pic = &s->picture[i];
1013         pic->reference = 3;
1014
1015         if (direct) {
1016             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
1017                 return ret;
1018         }
1019         ret = alloc_picture(s, pic, direct);
1020         if (ret < 0)
1021             return ret;
1022
1023         if (!direct) {
1024             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1025                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1026                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1027                 // empty
1028             } else {
1029                 int h_chroma_shift, v_chroma_shift;
1030                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
1031                                                  &h_chroma_shift,
1032                                                  &v_chroma_shift);
1033
1034                 for (i = 0; i < 3; i++) {
1035                     int src_stride = pic_arg->linesize[i];
1036                     int dst_stride = i ? s->uvlinesize : s->linesize;
1037                     int h_shift = i ? h_chroma_shift : 0;
1038                     int v_shift = i ? v_chroma_shift : 0;
1039                     int w = s->width  >> h_shift;
1040                     int h = s->height >> v_shift;
1041                     uint8_t *src = pic_arg->data[i];
1042                     uint8_t *dst = pic->f->data[i];
1043
1044                     if (!s->avctx->rc_buffer_size)
1045                         dst += INPLACE_OFFSET;
1046
1047                     if (src_stride == dst_stride)
1048                         memcpy(dst, src, src_stride * h);
1049                     else {
1050                         int h2 = h;
1051                         uint8_t *dst2 = dst;
1052                         while (h2--) {
1053                             memcpy(dst2, src, w);
1054                             dst2 += dst_stride;
1055                             src += src_stride;
1056                         }
1057                     }
1058                     if ((s->width & 15) || (s->height & 15)) {
1059                         s->mpvencdsp.draw_edges(dst, dst_stride,
1060                                                 w, h,
1061                                                 16 >> h_shift,
1062                                                 16 >> v_shift,
1063                                                 EDGE_BOTTOM);
1064                     }
1065                 }
1066             }
1067         }
1068         ret = av_frame_copy_props(pic->f, pic_arg);
1069         if (ret < 0)
1070             return ret;
1071
1072         pic->f->display_picture_number = display_picture_number;
1073         pic->f->pts = pts; // we set this here to avoid modifiying pic_arg
1074     }
1075
1076     /* shift buffer entries */
1077     for (i = 1; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1078         s->input_picture[i - 1] = s->input_picture[i];
1079
1080     s->input_picture[encoding_delay] = (Picture*) pic;
1081
1082     return 0;
1083 }
1084
1085 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1086 {
1087     int x, y, plane;
1088     int score = 0;
1089     int64_t score64 = 0;
1090
1091     for (plane = 0; plane < 3; plane++) {
1092         const int stride = p->f->linesize[plane];
1093         const int bw = plane ? 1 : 2;
1094         for (y = 0; y < s->mb_height * bw; y++) {
1095             for (x = 0; x < s->mb_width * bw; x++) {
1096                 int off = p->shared ? 0 : 16;
1097                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1098                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1099                 int v = s->mecc.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1100
1101                 switch (s->avctx->frame_skip_exp) {
1102                 case 0: score    =  FFMAX(score, v);          break;
1103                 case 1: score   += FFABS(v);                  break;
1104                 case 2: score   += v * v;                     break;
1105                 case 3: score64 += FFABS(v * v * (int64_t)v); break;
1106                 case 4: score64 += v * v * (int64_t)(v * v);  break;
1107                 }
1108             }
1109         }
1110     }
1111
1112     if (score)
1113         score64 = score;
1114
1115     if (score64 < s->avctx->frame_skip_threshold)
1116         return 1;
1117     if (score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda) >> 8))
1118         return 1;
1119     return 0;
1120 }
1121
1122 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1123 {
1124     AVPacket pkt = { 0 };
1125     int ret, got_output;
1126
1127     av_init_packet(&pkt);
1128     ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
1129     if (ret < 0)
1130         return ret;
1131
1132     ret = pkt.size;
1133     av_free_packet(&pkt);
1134     return ret;
1135 }
1136
1137 static int estimate_best_b_count(MpegEncContext *s)
1138 {
1139     AVCodec *codec    = avcodec_find_encoder(s->avctx->codec_id);
1140     AVCodecContext *c = avcodec_alloc_context3(NULL);
1141     const int scale = s->avctx->brd_scale;
1142     int i, j, out_size, p_lambda, b_lambda, lambda2;
1143     int64_t best_rd  = INT64_MAX;
1144     int best_b_count = -1;
1145
1146     if (!c)
1147         return AVERROR(ENOMEM);
1148     assert(scale >= 0 && scale <= 3);
1149
1150     //emms_c();
1151     //s->next_picture_ptr->quality;
1152     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1153     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1154     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1155     if (!b_lambda) // FIXME we should do this somewhere else
1156         b_lambda = p_lambda;
1157     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1158                FF_LAMBDA_SHIFT;
1159
1160     c->width        = s->width  >> scale;
1161     c->height       = s->height >> scale;
1162     c->flags        = CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR;
1163     c->flags       |= s->avctx->flags & CODEC_FLAG_QPEL;
1164     c->mb_decision  = s->avctx->mb_decision;
1165     c->me_cmp       = s->avctx->me_cmp;
1166     c->mb_cmp       = s->avctx->mb_cmp;
1167     c->me_sub_cmp   = s->avctx->me_sub_cmp;
1168     c->pix_fmt      = AV_PIX_FMT_YUV420P;
1169     c->time_base    = s->avctx->time_base;
1170     c->max_b_frames = s->max_b_frames;
1171
1172     if (avcodec_open2(c, codec, NULL) < 0)
1173         return -1;
1174
1175     for (i = 0; i < s->max_b_frames + 2; i++) {
1176         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1177                                                 s->next_picture_ptr;
1178
1179         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1180             pre_input = *pre_input_ptr;
1181
1182             if (!pre_input.shared && i) {
1183                 pre_input.f->data[0] += INPLACE_OFFSET;
1184                 pre_input.f->data[1] += INPLACE_OFFSET;
1185                 pre_input.f->data[2] += INPLACE_OFFSET;
1186             }
1187
1188             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[0],
1189                                        s->tmp_frames[i]->linesize[0],
1190                                        pre_input.f->data[0],
1191                                        pre_input.f->linesize[0],
1192                                        c->width, c->height);
1193             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[1],
1194                                        s->tmp_frames[i]->linesize[1],
1195                                        pre_input.f->data[1],
1196                                        pre_input.f->linesize[1],
1197                                        c->width >> 1, c->height >> 1);
1198             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[2],
1199                                        s->tmp_frames[i]->linesize[2],
1200                                        pre_input.f->data[2],
1201                                        pre_input.f->linesize[2],
1202                                        c->width >> 1, c->height >> 1);
1203         }
1204     }
1205
1206     for (j = 0; j < s->max_b_frames + 1; j++) {
1207         int64_t rd = 0;
1208
1209         if (!s->input_picture[j])
1210             break;
1211
1212         c->error[0] = c->error[1] = c->error[2] = 0;
1213
1214         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1215         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1216
1217         out_size = encode_frame(c, s->tmp_frames[0]);
1218
1219         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1220
1221         for (i = 0; i < s->max_b_frames + 1; i++) {
1222             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1223
1224             s->tmp_frames[i + 1]->pict_type = is_p ?
1225                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1226             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1227
1228             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1229
1230             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1231         }
1232
1233         /* get the delayed frames */
1234         while (out_size) {
1235             out_size = encode_frame(c, NULL);
1236             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1237         }
1238
1239         rd += c->error[0] + c->error[1] + c->error[2];
1240
1241         if (rd < best_rd) {
1242             best_rd = rd;
1243             best_b_count = j;
1244         }
1245     }
1246
1247     avcodec_close(c);
1248     av_freep(&c);
1249
1250     return best_b_count;
1251 }
1252
1253 static int select_input_picture(MpegEncContext *s)
1254 {
1255     int i, ret;
1256
1257     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1258         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1259     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1260
1261     /* set next picture type & ordering */
1262     if (!s->reordered_input_picture[0] && s->input_picture[0]) {
1263         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1264             !s->next_picture_ptr || s->intra_only) {
1265             s->reordered_input_picture[0] = s->input_picture[0];
1266             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1267             s->reordered_input_picture[0]->f->coded_picture_number =
1268                 s->coded_picture_number++;
1269         } else {
1270             int b_frames;
1271
1272             if (s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor) {
1273                 if (s->picture_in_gop_number < s->gop_size &&
1274                     skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1275                     // FIXME check that te gop check above is +-1 correct
1276                     av_frame_unref(s->input_picture[0]->f);
1277
1278                     emms_c();
1279                     ff_vbv_update(s, 0);
1280
1281                     goto no_output_pic;
1282                 }
1283             }
1284
1285             if (s->avctx->flags & CODEC_FLAG_PASS2) {
1286                 for (i = 0; i < s->max_b_frames + 1; i++) {
1287                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1288
1289                     if (pict_num >= s->rc_context.num_entries)
1290                         break;
1291                     if (!s->input_picture[i]) {
1292                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1293                         break;
1294                     }
1295
1296                     s->input_picture[i]->f->pict_type =
1297                         s->rc_context.entry[pict_num].new_pict_type;
1298                 }
1299             }
1300
1301             if (s->avctx->b_frame_strategy == 0) {
1302                 b_frames = s->max_b_frames;
1303                 while (b_frames && !s->input_picture[b_frames])
1304                     b_frames--;
1305             } else if (s->avctx->b_frame_strategy == 1) {
1306                 for (i = 1; i < s->max_b_frames + 1; i++) {
1307                     if (s->input_picture[i] &&
1308                         s->input_picture[i]->b_frame_score == 0) {
1309                         s->input_picture[i]->b_frame_score =
1310                             get_intra_count(s,
1311                                             s->input_picture[i    ]->f->data[0],
1312                                             s->input_picture[i - 1]->f->data[0],
1313                                             s->linesize) + 1;
1314                     }
1315                 }
1316                 for (i = 0; i < s->max_b_frames + 1; i++) {
1317                     if (!s->input_picture[i] ||
1318                         s->input_picture[i]->b_frame_score - 1 >
1319                             s->mb_num / s->avctx->b_sensitivity)
1320                         break;
1321                 }
1322
1323                 b_frames = FFMAX(0, i - 1);
1324
1325                 /* reset scores */
1326                 for (i = 0; i < b_frames + 1; i++) {
1327                     s->input_picture[i]->b_frame_score = 0;
1328                 }
1329             } else if (s->avctx->b_frame_strategy == 2) {
1330                 b_frames = estimate_best_b_count(s);
1331             } else {
1332                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1333                 b_frames = 0;
1334             }
1335
1336             emms_c();
1337
1338             for (i = b_frames - 1; i >= 0; i--) {
1339                 int type = s->input_picture[i]->f->pict_type;
1340                 if (type && type != AV_PICTURE_TYPE_B)
1341                     b_frames = i;
1342             }
1343             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1344                 b_frames == s->max_b_frames) {
1345                 av_log(s->avctx, AV_LOG_ERROR,
1346                        "warning, too many b frames in a row\n");
1347             }
1348
1349             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1350                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1351                     s->gop_size > s->picture_in_gop_number) {
1352                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1353                 } else {
1354                     if (s->avctx->flags & CODEC_FLAG_CLOSED_GOP)
1355                         b_frames = 0;
1356                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1357                 }
1358             }
1359
1360             if ((s->avctx->flags & CODEC_FLAG_CLOSED_GOP) && b_frames &&
1361                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1362                 b_frames--;
1363
1364             s->reordered_input_picture[0] = s->input_picture[b_frames];
1365             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1366                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1367             s->reordered_input_picture[0]->f->coded_picture_number =
1368                 s->coded_picture_number++;
1369             for (i = 0; i < b_frames; i++) {
1370                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1371                 s->reordered_input_picture[i + 1]->f->pict_type =
1372                     AV_PICTURE_TYPE_B;
1373                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1374                     s->coded_picture_number++;
1375             }
1376         }
1377     }
1378 no_output_pic:
1379     if (s->reordered_input_picture[0]) {
1380         s->reordered_input_picture[0]->reference =
1381            s->reordered_input_picture[0]->f->pict_type !=
1382                AV_PICTURE_TYPE_B ? 3 : 0;
1383
1384         ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1385         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->new_picture, s->reordered_input_picture[0])))
1386             return ret;
1387
1388         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1389             // input is a shared pix, so we can't modifiy it -> alloc a new
1390             // one & ensure that the shared one is reuseable
1391
1392             Picture *pic;
1393             int i = ff_find_unused_picture(s->avctx, s->picture, 0);
1394             if (i < 0)
1395                 return i;
1396             pic = &s->picture[i];
1397
1398             pic->reference = s->reordered_input_picture[0]->reference;
1399             if (alloc_picture(s, pic, 0) < 0) {
1400                 return -1;
1401             }
1402
1403             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1404             if (ret < 0)
1405                 return ret;
1406
1407             /* mark us unused / free shared pic */
1408             av_frame_unref(s->reordered_input_picture[0]->f);
1409             s->reordered_input_picture[0]->shared = 0;
1410
1411             s->current_picture_ptr = pic;
1412         } else {
1413             // input is not a shared pix -> reuse buffer for current_pix
1414             s->current_picture_ptr = s->reordered_input_picture[0];
1415             for (i = 0; i < 4; i++) {
1416                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1417             }
1418         }
1419         ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1420         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1421                                        s->current_picture_ptr)) < 0)
1422             return ret;
1423
1424         s->picture_number = s->new_picture.f->display_picture_number;
1425     } else {
1426         ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1427     }
1428     return 0;
1429 }
1430
1431 static void frame_end(MpegEncContext *s)
1432 {
1433     int i;
1434
1435     if (s->unrestricted_mv &&
1436         s->current_picture.reference &&
1437         !s->intra_only) {
1438         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1439         int hshift = desc->log2_chroma_w;
1440         int vshift = desc->log2_chroma_h;
1441         s->mpvencdsp.draw_edges(s->current_picture.f->data[0], s->linesize,
1442                                 s->h_edge_pos, s->v_edge_pos,
1443                                 EDGE_WIDTH, EDGE_WIDTH,
1444                                 EDGE_TOP | EDGE_BOTTOM);
1445         s->mpvencdsp.draw_edges(s->current_picture.f->data[1], s->uvlinesize,
1446                                 s->h_edge_pos >> hshift,
1447                                 s->v_edge_pos >> vshift,
1448                                 EDGE_WIDTH >> hshift,
1449                                 EDGE_WIDTH >> vshift,
1450                                 EDGE_TOP | EDGE_BOTTOM);
1451         s->mpvencdsp.draw_edges(s->current_picture.f->data[2], s->uvlinesize,
1452                                 s->h_edge_pos >> hshift,
1453                                 s->v_edge_pos >> vshift,
1454                                 EDGE_WIDTH >> hshift,
1455                                 EDGE_WIDTH >> vshift,
1456                                 EDGE_TOP | EDGE_BOTTOM);
1457     }
1458
1459     emms_c();
1460
1461     s->last_pict_type                 = s->pict_type;
1462     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1463     if (s->pict_type!= AV_PICTURE_TYPE_B)
1464         s->last_non_b_pict_type = s->pict_type;
1465
1466     if (s->encoding) {
1467         /* release non-reference frames */
1468         for (i = 0; i < MAX_PICTURE_COUNT; i++) {
1469             if (!s->picture[i].reference)
1470                 ff_mpeg_unref_picture(s->avctx, &s->picture[i]);
1471         }
1472     }
1473
1474     s->avctx->coded_frame = s->current_picture_ptr->f;
1475
1476 }
1477
1478 static void update_noise_reduction(MpegEncContext *s)
1479 {
1480     int intra, i;
1481
1482     for (intra = 0; intra < 2; intra++) {
1483         if (s->dct_count[intra] > (1 << 16)) {
1484             for (i = 0; i < 64; i++) {
1485                 s->dct_error_sum[intra][i] >>= 1;
1486             }
1487             s->dct_count[intra] >>= 1;
1488         }
1489
1490         for (i = 0; i < 64; i++) {
1491             s->dct_offset[intra][i] = (s->avctx->noise_reduction *
1492                                        s->dct_count[intra] +
1493                                        s->dct_error_sum[intra][i] / 2) /
1494                                       (s->dct_error_sum[intra][i] + 1);
1495         }
1496     }
1497 }
1498
1499 static int frame_start(MpegEncContext *s)
1500 {
1501     int ret;
1502
1503     /* mark & release old frames */
1504     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1505         s->last_picture_ptr != s->next_picture_ptr &&
1506         s->last_picture_ptr->f->buf[0]) {
1507         ff_mpeg_unref_picture(s->avctx, s->last_picture_ptr);
1508     }
1509
1510     s->current_picture_ptr->f->pict_type = s->pict_type;
1511     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1512
1513     ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1514     if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1515                                    s->current_picture_ptr)) < 0)
1516         return ret;
1517
1518     if (s->pict_type != AV_PICTURE_TYPE_B) {
1519         s->last_picture_ptr = s->next_picture_ptr;
1520         if (!s->droppable)
1521             s->next_picture_ptr = s->current_picture_ptr;
1522     }
1523
1524     if (s->last_picture_ptr) {
1525         ff_mpeg_unref_picture(s->avctx, &s->last_picture);
1526         if (s->last_picture_ptr->f->buf[0] &&
1527             (ret = ff_mpeg_ref_picture(s->avctx, &s->last_picture,
1528                                        s->last_picture_ptr)) < 0)
1529             return ret;
1530     }
1531     if (s->next_picture_ptr) {
1532         ff_mpeg_unref_picture(s->avctx, &s->next_picture);
1533         if (s->next_picture_ptr->f->buf[0] &&
1534             (ret = ff_mpeg_ref_picture(s->avctx, &s->next_picture,
1535                                        s->next_picture_ptr)) < 0)
1536             return ret;
1537     }
1538
1539     if (s->picture_structure!= PICT_FRAME) {
1540         int i;
1541         for (i = 0; i < 4; i++) {
1542             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1543                 s->current_picture.f->data[i] +=
1544                     s->current_picture.f->linesize[i];
1545             }
1546             s->current_picture.f->linesize[i] *= 2;
1547             s->last_picture.f->linesize[i]    *= 2;
1548             s->next_picture.f->linesize[i]    *= 2;
1549         }
1550     }
1551
1552     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1553         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1554         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1555     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1556         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1557         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1558     } else {
1559         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1560         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1561     }
1562
1563     if (s->dct_error_sum) {
1564         assert(s->avctx->noise_reduction && s->encoding);
1565         update_noise_reduction(s);
1566     }
1567
1568     return 0;
1569 }
1570
1571 int ff_mpv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1572                           const AVFrame *pic_arg, int *got_packet)
1573 {
1574     MpegEncContext *s = avctx->priv_data;
1575     int i, stuffing_count, ret;
1576     int context_count = s->slice_context_count;
1577
1578     s->picture_in_gop_number++;
1579
1580     if (load_input_picture(s, pic_arg) < 0)
1581         return -1;
1582
1583     if (select_input_picture(s) < 0) {
1584         return -1;
1585     }
1586
1587     /* output? */
1588     if (s->new_picture.f->data[0]) {
1589         if (!pkt->data &&
1590             (ret = ff_alloc_packet(pkt, s->mb_width*s->mb_height*MAX_MB_BYTES)) < 0)
1591             return ret;
1592         if (s->mb_info) {
1593             s->mb_info_ptr = av_packet_new_side_data(pkt,
1594                                  AV_PKT_DATA_H263_MB_INFO,
1595                                  s->mb_width*s->mb_height*12);
1596             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1597         }
1598
1599         for (i = 0; i < context_count; i++) {
1600             int start_y = s->thread_context[i]->start_mb_y;
1601             int   end_y = s->thread_context[i]->  end_mb_y;
1602             int h       = s->mb_height;
1603             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1604             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1605
1606             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1607         }
1608
1609         s->pict_type = s->new_picture.f->pict_type;
1610         //emms_c();
1611         ret = frame_start(s);
1612         if (ret < 0)
1613             return ret;
1614 vbv_retry:
1615         if (encode_picture(s, s->picture_number) < 0)
1616             return -1;
1617
1618         avctx->header_bits = s->header_bits;
1619         avctx->mv_bits     = s->mv_bits;
1620         avctx->misc_bits   = s->misc_bits;
1621         avctx->i_tex_bits  = s->i_tex_bits;
1622         avctx->p_tex_bits  = s->p_tex_bits;
1623         avctx->i_count     = s->i_count;
1624         // FIXME f/b_count in avctx
1625         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1626         avctx->skip_count  = s->skip_count;
1627
1628         frame_end(s);
1629
1630         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1631             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1632
1633         if (avctx->rc_buffer_size) {
1634             RateControlContext *rcc = &s->rc_context;
1635             int max_size = rcc->buffer_index * avctx->rc_max_available_vbv_use;
1636
1637             if (put_bits_count(&s->pb) > max_size &&
1638                 s->lambda < s->lmax) {
1639                 s->next_lambda = FFMAX(s->lambda + 1, s->lambda *
1640                                        (s->qscale + 1) / s->qscale);
1641                 if (s->adaptive_quant) {
1642                     int i;
1643                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1644                         s->lambda_table[i] =
1645                             FFMAX(s->lambda_table[i] + 1,
1646                                   s->lambda_table[i] * (s->qscale + 1) /
1647                                   s->qscale);
1648                 }
1649                 s->mb_skipped = 0;        // done in frame_start()
1650                 // done in encode_picture() so we must undo it
1651                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1652                     if (s->flipflop_rounding          ||
1653                         s->codec_id == AV_CODEC_ID_H263P ||
1654                         s->codec_id == AV_CODEC_ID_MPEG4)
1655                         s->no_rounding ^= 1;
1656                 }
1657                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1658                     s->time_base       = s->last_time_base;
1659                     s->last_non_b_time = s->time - s->pp_time;
1660                 }
1661                 for (i = 0; i < context_count; i++) {
1662                     PutBitContext *pb = &s->thread_context[i]->pb;
1663                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1664                 }
1665                 goto vbv_retry;
1666             }
1667
1668             assert(s->avctx->rc_max_rate);
1669         }
1670
1671         if (s->avctx->flags & CODEC_FLAG_PASS1)
1672             ff_write_pass1_stats(s);
1673
1674         for (i = 0; i < 4; i++) {
1675             s->current_picture_ptr->f->error[i] = s->current_picture.f->error[i];
1676             avctx->error[i] += s->current_picture_ptr->f->error[i];
1677         }
1678
1679         if (s->avctx->flags & CODEC_FLAG_PASS1)
1680             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits +
1681                    avctx->i_tex_bits + avctx->p_tex_bits ==
1682                        put_bits_count(&s->pb));
1683         flush_put_bits(&s->pb);
1684         s->frame_bits  = put_bits_count(&s->pb);
1685
1686         stuffing_count = ff_vbv_update(s, s->frame_bits);
1687         if (stuffing_count) {
1688             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1689                     stuffing_count + 50) {
1690                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1691                 return -1;
1692             }
1693
1694             switch (s->codec_id) {
1695             case AV_CODEC_ID_MPEG1VIDEO:
1696             case AV_CODEC_ID_MPEG2VIDEO:
1697                 while (stuffing_count--) {
1698                     put_bits(&s->pb, 8, 0);
1699                 }
1700             break;
1701             case AV_CODEC_ID_MPEG4:
1702                 put_bits(&s->pb, 16, 0);
1703                 put_bits(&s->pb, 16, 0x1C3);
1704                 stuffing_count -= 4;
1705                 while (stuffing_count--) {
1706                     put_bits(&s->pb, 8, 0xFF);
1707                 }
1708             break;
1709             default:
1710                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1711             }
1712             flush_put_bits(&s->pb);
1713             s->frame_bits  = put_bits_count(&s->pb);
1714         }
1715
1716         /* update mpeg1/2 vbv_delay for CBR */
1717         if (s->avctx->rc_max_rate                          &&
1718             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1719             s->out_format == FMT_MPEG1                     &&
1720             90000LL * (avctx->rc_buffer_size - 1) <=
1721                 s->avctx->rc_max_rate * 0xFFFFLL) {
1722             int vbv_delay, min_delay;
1723             double inbits  = s->avctx->rc_max_rate *
1724                              av_q2d(s->avctx->time_base);
1725             int    minbits = s->frame_bits - 8 *
1726                              (s->vbv_delay_ptr - s->pb.buf - 1);
1727             double bits    = s->rc_context.buffer_index + minbits - inbits;
1728
1729             if (bits < 0)
1730                 av_log(s->avctx, AV_LOG_ERROR,
1731                        "Internal error, negative bits\n");
1732
1733             assert(s->repeat_first_field == 0);
1734
1735             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
1736             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
1737                         s->avctx->rc_max_rate;
1738
1739             vbv_delay = FFMAX(vbv_delay, min_delay);
1740
1741             assert(vbv_delay < 0xFFFF);
1742
1743             s->vbv_delay_ptr[0] &= 0xF8;
1744             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
1745             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
1746             s->vbv_delay_ptr[2] &= 0x07;
1747             s->vbv_delay_ptr[2] |= vbv_delay << 3;
1748             avctx->vbv_delay     = vbv_delay * 300;
1749         }
1750         s->total_bits     += s->frame_bits;
1751         avctx->frame_bits  = s->frame_bits;
1752
1753         pkt->pts = s->current_picture.f->pts;
1754         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
1755             if (!s->current_picture.f->coded_picture_number)
1756                 pkt->dts = pkt->pts - s->dts_delta;
1757             else
1758                 pkt->dts = s->reordered_pts;
1759             s->reordered_pts = pkt->pts;
1760         } else
1761             pkt->dts = pkt->pts;
1762         if (s->current_picture.f->key_frame)
1763             pkt->flags |= AV_PKT_FLAG_KEY;
1764         if (s->mb_info)
1765             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
1766     } else {
1767         s->frame_bits = 0;
1768     }
1769     assert((s->frame_bits & 7) == 0);
1770
1771     pkt->size = s->frame_bits / 8;
1772     *got_packet = !!pkt->size;
1773     return 0;
1774 }
1775
1776 static inline void dct_single_coeff_elimination(MpegEncContext *s,
1777                                                 int n, int threshold)
1778 {
1779     static const char tab[64] = {
1780         3, 2, 2, 1, 1, 1, 1, 1,
1781         1, 1, 1, 1, 1, 1, 1, 1,
1782         1, 1, 1, 1, 1, 1, 1, 1,
1783         0, 0, 0, 0, 0, 0, 0, 0,
1784         0, 0, 0, 0, 0, 0, 0, 0,
1785         0, 0, 0, 0, 0, 0, 0, 0,
1786         0, 0, 0, 0, 0, 0, 0, 0,
1787         0, 0, 0, 0, 0, 0, 0, 0
1788     };
1789     int score = 0;
1790     int run = 0;
1791     int i;
1792     int16_t *block = s->block[n];
1793     const int last_index = s->block_last_index[n];
1794     int skip_dc;
1795
1796     if (threshold < 0) {
1797         skip_dc = 0;
1798         threshold = -threshold;
1799     } else
1800         skip_dc = 1;
1801
1802     /* Are all we could set to zero already zero? */
1803     if (last_index <= skip_dc - 1)
1804         return;
1805
1806     for (i = 0; i <= last_index; i++) {
1807         const int j = s->intra_scantable.permutated[i];
1808         const int level = FFABS(block[j]);
1809         if (level == 1) {
1810             if (skip_dc && i == 0)
1811                 continue;
1812             score += tab[run];
1813             run = 0;
1814         } else if (level > 1) {
1815             return;
1816         } else {
1817             run++;
1818         }
1819     }
1820     if (score >= threshold)
1821         return;
1822     for (i = skip_dc; i <= last_index; i++) {
1823         const int j = s->intra_scantable.permutated[i];
1824         block[j] = 0;
1825     }
1826     if (block[0])
1827         s->block_last_index[n] = 0;
1828     else
1829         s->block_last_index[n] = -1;
1830 }
1831
1832 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
1833                                int last_index)
1834 {
1835     int i;
1836     const int maxlevel = s->max_qcoeff;
1837     const int minlevel = s->min_qcoeff;
1838     int overflow = 0;
1839
1840     if (s->mb_intra) {
1841         i = 1; // skip clipping of intra dc
1842     } else
1843         i = 0;
1844
1845     for (; i <= last_index; i++) {
1846         const int j = s->intra_scantable.permutated[i];
1847         int level = block[j];
1848
1849         if (level > maxlevel) {
1850             level = maxlevel;
1851             overflow++;
1852         } else if (level < minlevel) {
1853             level = minlevel;
1854             overflow++;
1855         }
1856
1857         block[j] = level;
1858     }
1859
1860     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1861         av_log(s->avctx, AV_LOG_INFO,
1862                "warning, clipping %d dct coefficients to %d..%d\n",
1863                overflow, minlevel, maxlevel);
1864 }
1865
1866 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
1867 {
1868     int x, y;
1869     // FIXME optimize
1870     for (y = 0; y < 8; y++) {
1871         for (x = 0; x < 8; x++) {
1872             int x2, y2;
1873             int sum = 0;
1874             int sqr = 0;
1875             int count = 0;
1876
1877             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
1878                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
1879                     int v = ptr[x2 + y2 * stride];
1880                     sum += v;
1881                     sqr += v * v;
1882                     count++;
1883                 }
1884             }
1885             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
1886         }
1887     }
1888 }
1889
1890 static av_always_inline void encode_mb_internal(MpegEncContext *s,
1891                                                 int motion_x, int motion_y,
1892                                                 int mb_block_height,
1893                                                 int mb_block_count)
1894 {
1895     int16_t weight[8][64];
1896     int16_t orig[8][64];
1897     const int mb_x = s->mb_x;
1898     const int mb_y = s->mb_y;
1899     int i;
1900     int skip_dct[8];
1901     int dct_offset = s->linesize * 8; // default for progressive frames
1902     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1903     ptrdiff_t wrap_y, wrap_c;
1904
1905     for (i = 0; i < mb_block_count; i++)
1906         skip_dct[i] = s->skipdct;
1907
1908     if (s->adaptive_quant) {
1909         const int last_qp = s->qscale;
1910         const int mb_xy = mb_x + mb_y * s->mb_stride;
1911
1912         s->lambda = s->lambda_table[mb_xy];
1913         update_qscale(s);
1914
1915         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
1916             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
1917             s->dquant = s->qscale - last_qp;
1918
1919             if (s->out_format == FMT_H263) {
1920                 s->dquant = av_clip(s->dquant, -2, 2);
1921
1922                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
1923                     if (!s->mb_intra) {
1924                         if (s->pict_type == AV_PICTURE_TYPE_B) {
1925                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
1926                                 s->dquant = 0;
1927                         }
1928                         if (s->mv_type == MV_TYPE_8X8)
1929                             s->dquant = 0;
1930                     }
1931                 }
1932             }
1933         }
1934         ff_set_qscale(s, last_qp + s->dquant);
1935     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
1936         ff_set_qscale(s, s->qscale + s->dquant);
1937
1938     wrap_y = s->linesize;
1939     wrap_c = s->uvlinesize;
1940     ptr_y  = s->new_picture.f->data[0] +
1941              (mb_y * 16 * wrap_y)              + mb_x * 16;
1942     ptr_cb = s->new_picture.f->data[1] +
1943              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1944     ptr_cr = s->new_picture.f->data[2] +
1945              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1946
1947     if (mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) {
1948         uint8_t *ebuf = s->sc.edge_emu_buffer + 32;
1949         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
1950                                  wrap_y, wrap_y,
1951                                  16, 16, mb_x * 16, mb_y * 16,
1952                                  s->width, s->height);
1953         ptr_y = ebuf;
1954         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y, ptr_cb,
1955                                  wrap_c, wrap_c,
1956                                  8, mb_block_height, mb_x * 8, mb_y * 8,
1957                                  s->width >> 1, s->height >> 1);
1958         ptr_cb = ebuf + 18 * wrap_y;
1959         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y + 8, ptr_cr,
1960                                  wrap_c, wrap_c,
1961                                  8, mb_block_height, mb_x * 8, mb_y * 8,
1962                                  s->width >> 1, s->height >> 1);
1963         ptr_cr = ebuf + 18 * wrap_y + 8;
1964     }
1965
1966     if (s->mb_intra) {
1967         if (s->avctx->flags & CODEC_FLAG_INTERLACED_DCT) {
1968             int progressive_score, interlaced_score;
1969
1970             s->interlaced_dct = 0;
1971             progressive_score = s->mecc.ildct_cmp[4](s, ptr_y, NULL, wrap_y, 8) +
1972                                 s->mecc.ildct_cmp[4](s, ptr_y + wrap_y * 8,
1973                                                      NULL, wrap_y, 8) - 400;
1974
1975             if (progressive_score > 0) {
1976                 interlaced_score = s->mecc.ildct_cmp[4](s, ptr_y,
1977                                                         NULL, wrap_y * 2, 8) +
1978                                    s->mecc.ildct_cmp[4](s, ptr_y + wrap_y,
1979                                                         NULL, wrap_y * 2, 8);
1980                 if (progressive_score > interlaced_score) {
1981                     s->interlaced_dct = 1;
1982
1983                     dct_offset = wrap_y;
1984                     wrap_y <<= 1;
1985                     if (s->chroma_format == CHROMA_422)
1986                         wrap_c <<= 1;
1987                 }
1988             }
1989         }
1990
1991         s->pdsp.get_pixels(s->block[0], ptr_y,                  wrap_y);
1992         s->pdsp.get_pixels(s->block[1], ptr_y + 8,              wrap_y);
1993         s->pdsp.get_pixels(s->block[2], ptr_y + dct_offset,     wrap_y);
1994         s->pdsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
1995
1996         if (s->avctx->flags & CODEC_FLAG_GRAY) {
1997             skip_dct[4] = 1;
1998             skip_dct[5] = 1;
1999         } else {
2000             s->pdsp.get_pixels(s->block[4], ptr_cb, wrap_c);
2001             s->pdsp.get_pixels(s->block[5], ptr_cr, wrap_c);
2002             if (!s->chroma_y_shift) { /* 422 */
2003                 s->pdsp.get_pixels(s->block[6],
2004                                    ptr_cb + (dct_offset >> 1), wrap_c);
2005                 s->pdsp.get_pixels(s->block[7],
2006                                    ptr_cr + (dct_offset >> 1), wrap_c);
2007             }
2008         }
2009     } else {
2010         op_pixels_func (*op_pix)[4];
2011         qpel_mc_func (*op_qpix)[16];
2012         uint8_t *dest_y, *dest_cb, *dest_cr;
2013
2014         dest_y  = s->dest[0];
2015         dest_cb = s->dest[1];
2016         dest_cr = s->dest[2];
2017
2018         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
2019             op_pix  = s->hdsp.put_pixels_tab;
2020             op_qpix = s->qdsp.put_qpel_pixels_tab;
2021         } else {
2022             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
2023             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
2024         }
2025
2026         if (s->mv_dir & MV_DIR_FORWARD) {
2027             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 0,
2028                           s->last_picture.f->data,
2029                           op_pix, op_qpix);
2030             op_pix  = s->hdsp.avg_pixels_tab;
2031             op_qpix = s->qdsp.avg_qpel_pixels_tab;
2032         }
2033         if (s->mv_dir & MV_DIR_BACKWARD) {
2034             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 1,
2035                           s->next_picture.f->data,
2036                           op_pix, op_qpix);
2037         }
2038
2039         if (s->avctx->flags & CODEC_FLAG_INTERLACED_DCT) {
2040             int progressive_score, interlaced_score;
2041
2042             s->interlaced_dct = 0;
2043             progressive_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y, wrap_y, 8) +
2044                                 s->mecc.ildct_cmp[0](s, dest_y + wrap_y * 8,
2045                                                      ptr_y + wrap_y * 8,
2046                                                      wrap_y, 8) - 400;
2047
2048             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2049                 progressive_score -= 400;
2050
2051             if (progressive_score > 0) {
2052                 interlaced_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y,
2053                                                         wrap_y * 2, 8) +
2054                                    s->mecc.ildct_cmp[0](s, dest_y + wrap_y,
2055                                                         ptr_y + wrap_y,
2056                                                         wrap_y * 2, 8);
2057
2058                 if (progressive_score > interlaced_score) {
2059                     s->interlaced_dct = 1;
2060
2061                     dct_offset = wrap_y;
2062                     wrap_y <<= 1;
2063                     if (s->chroma_format == CHROMA_422)
2064                         wrap_c <<= 1;
2065                 }
2066             }
2067         }
2068
2069         s->pdsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2070         s->pdsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2071         s->pdsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2072                             dest_y + dct_offset, wrap_y);
2073         s->pdsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2074                             dest_y + dct_offset + 8, wrap_y);
2075
2076         if (s->avctx->flags & CODEC_FLAG_GRAY) {
2077             skip_dct[4] = 1;
2078             skip_dct[5] = 1;
2079         } else {
2080             s->pdsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2081             s->pdsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2082             if (!s->chroma_y_shift) { /* 422 */
2083                 s->pdsp.diff_pixels(s->block[6], ptr_cb + (dct_offset >> 1),
2084                                     dest_cb + (dct_offset >> 1), wrap_c);
2085                 s->pdsp.diff_pixels(s->block[7], ptr_cr + (dct_offset >> 1),
2086                                     dest_cr + (dct_offset >> 1), wrap_c);
2087             }
2088         }
2089         /* pre quantization */
2090         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2091                 2 * s->qscale * s->qscale) {
2092             // FIXME optimize
2093             if (s->mecc.sad[1](NULL, ptr_y, dest_y, wrap_y, 8) < 20 * s->qscale)
2094                 skip_dct[0] = 1;
2095             if (s->mecc.sad[1](NULL, ptr_y + 8, dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2096                 skip_dct[1] = 1;
2097             if (s->mecc.sad[1](NULL, ptr_y + dct_offset, dest_y + dct_offset,
2098                                wrap_y, 8) < 20 * s->qscale)
2099                 skip_dct[2] = 1;
2100             if (s->mecc.sad[1](NULL, ptr_y + dct_offset + 8, dest_y + dct_offset + 8,
2101                                wrap_y, 8) < 20 * s->qscale)
2102                 skip_dct[3] = 1;
2103             if (s->mecc.sad[1](NULL, ptr_cb, dest_cb, wrap_c, 8) < 20 * s->qscale)
2104                 skip_dct[4] = 1;
2105             if (s->mecc.sad[1](NULL, ptr_cr, dest_cr, wrap_c, 8) < 20 * s->qscale)
2106                 skip_dct[5] = 1;
2107             if (!s->chroma_y_shift) { /* 422 */
2108                 if (s->mecc.sad[1](NULL, ptr_cb + (dct_offset >> 1),
2109                                    dest_cb + (dct_offset >> 1),
2110                                    wrap_c, 8) < 20 * s->qscale)
2111                     skip_dct[6] = 1;
2112                 if (s->mecc.sad[1](NULL, ptr_cr + (dct_offset >> 1),
2113                                    dest_cr + (dct_offset >> 1),
2114                                    wrap_c, 8) < 20 * s->qscale)
2115                     skip_dct[7] = 1;
2116             }
2117         }
2118     }
2119
2120     if (s->quantizer_noise_shaping) {
2121         if (!skip_dct[0])
2122             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2123         if (!skip_dct[1])
2124             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2125         if (!skip_dct[2])
2126             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2127         if (!skip_dct[3])
2128             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2129         if (!skip_dct[4])
2130             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2131         if (!skip_dct[5])
2132             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2133         if (!s->chroma_y_shift) { /* 422 */
2134             if (!skip_dct[6])
2135                 get_visual_weight(weight[6], ptr_cb + (dct_offset >> 1),
2136                                   wrap_c);
2137             if (!skip_dct[7])
2138                 get_visual_weight(weight[7], ptr_cr + (dct_offset >> 1),
2139                                   wrap_c);
2140         }
2141         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2142     }
2143
2144     /* DCT & quantize */
2145     assert(s->out_format != FMT_MJPEG || s->qscale == 8);
2146     {
2147         for (i = 0; i < mb_block_count; i++) {
2148             if (!skip_dct[i]) {
2149                 int overflow;
2150                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2151                 // FIXME we could decide to change to quantizer instead of
2152                 // clipping
2153                 // JS: I don't think that would be a good idea it could lower
2154                 //     quality instead of improve it. Just INTRADC clipping
2155                 //     deserves changes in quantizer
2156                 if (overflow)
2157                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2158             } else
2159                 s->block_last_index[i] = -1;
2160         }
2161         if (s->quantizer_noise_shaping) {
2162             for (i = 0; i < mb_block_count; i++) {
2163                 if (!skip_dct[i]) {
2164                     s->block_last_index[i] =
2165                         dct_quantize_refine(s, s->block[i], weight[i],
2166                                             orig[i], i, s->qscale);
2167                 }
2168             }
2169         }
2170
2171         if (s->luma_elim_threshold && !s->mb_intra)
2172             for (i = 0; i < 4; i++)
2173                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2174         if (s->chroma_elim_threshold && !s->mb_intra)
2175             for (i = 4; i < mb_block_count; i++)
2176                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2177
2178         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2179             for (i = 0; i < mb_block_count; i++) {
2180                 if (s->block_last_index[i] == -1)
2181                     s->coded_score[i] = INT_MAX / 256;
2182             }
2183         }
2184     }
2185
2186     if ((s->avctx->flags & CODEC_FLAG_GRAY) && s->mb_intra) {
2187         s->block_last_index[4] =
2188         s->block_last_index[5] = 0;
2189         s->block[4][0] =
2190         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2191     }
2192
2193     // non c quantize code returns incorrect block_last_index FIXME
2194     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2195         for (i = 0; i < mb_block_count; i++) {
2196             int j;
2197             if (s->block_last_index[i] > 0) {
2198                 for (j = 63; j > 0; j--) {
2199                     if (s->block[i][s->intra_scantable.permutated[j]])
2200                         break;
2201                 }
2202                 s->block_last_index[i] = j;
2203             }
2204         }
2205     }
2206
2207     /* huffman encode */
2208     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2209     case AV_CODEC_ID_MPEG1VIDEO:
2210     case AV_CODEC_ID_MPEG2VIDEO:
2211         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2212             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2213         break;
2214     case AV_CODEC_ID_MPEG4:
2215         if (CONFIG_MPEG4_ENCODER)
2216             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2217         break;
2218     case AV_CODEC_ID_MSMPEG4V2:
2219     case AV_CODEC_ID_MSMPEG4V3:
2220     case AV_CODEC_ID_WMV1:
2221         if (CONFIG_MSMPEG4_ENCODER)
2222             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2223         break;
2224     case AV_CODEC_ID_WMV2:
2225         if (CONFIG_WMV2_ENCODER)
2226             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2227         break;
2228     case AV_CODEC_ID_H261:
2229         if (CONFIG_H261_ENCODER)
2230             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2231         break;
2232     case AV_CODEC_ID_H263:
2233     case AV_CODEC_ID_H263P:
2234     case AV_CODEC_ID_FLV1:
2235     case AV_CODEC_ID_RV10:
2236     case AV_CODEC_ID_RV20:
2237         if (CONFIG_H263_ENCODER)
2238             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2239         break;
2240     case AV_CODEC_ID_MJPEG:
2241         if (CONFIG_MJPEG_ENCODER)
2242             ff_mjpeg_encode_mb(s, s->block);
2243         break;
2244     default:
2245         assert(0);
2246     }
2247 }
2248
2249 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2250 {
2251     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
2252     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
2253 }
2254
2255 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2256     int i;
2257
2258     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2259
2260     /* mpeg1 */
2261     d->mb_skip_run= s->mb_skip_run;
2262     for(i=0; i<3; i++)
2263         d->last_dc[i] = s->last_dc[i];
2264
2265     /* statistics */
2266     d->mv_bits= s->mv_bits;
2267     d->i_tex_bits= s->i_tex_bits;
2268     d->p_tex_bits= s->p_tex_bits;
2269     d->i_count= s->i_count;
2270     d->f_count= s->f_count;
2271     d->b_count= s->b_count;
2272     d->skip_count= s->skip_count;
2273     d->misc_bits= s->misc_bits;
2274     d->last_bits= 0;
2275
2276     d->mb_skipped= 0;
2277     d->qscale= s->qscale;
2278     d->dquant= s->dquant;
2279
2280     d->esc3_level_length= s->esc3_level_length;
2281 }
2282
2283 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2284     int i;
2285
2286     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2287     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2288
2289     /* mpeg1 */
2290     d->mb_skip_run= s->mb_skip_run;
2291     for(i=0; i<3; i++)
2292         d->last_dc[i] = s->last_dc[i];
2293
2294     /* statistics */
2295     d->mv_bits= s->mv_bits;
2296     d->i_tex_bits= s->i_tex_bits;
2297     d->p_tex_bits= s->p_tex_bits;
2298     d->i_count= s->i_count;
2299     d->f_count= s->f_count;
2300     d->b_count= s->b_count;
2301     d->skip_count= s->skip_count;
2302     d->misc_bits= s->misc_bits;
2303
2304     d->mb_intra= s->mb_intra;
2305     d->mb_skipped= s->mb_skipped;
2306     d->mv_type= s->mv_type;
2307     d->mv_dir= s->mv_dir;
2308     d->pb= s->pb;
2309     if(s->data_partitioning){
2310         d->pb2= s->pb2;
2311         d->tex_pb= s->tex_pb;
2312     }
2313     d->block= s->block;
2314     for(i=0; i<8; i++)
2315         d->block_last_index[i]= s->block_last_index[i];
2316     d->interlaced_dct= s->interlaced_dct;
2317     d->qscale= s->qscale;
2318
2319     d->esc3_level_length= s->esc3_level_length;
2320 }
2321
2322 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2323                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2324                            int *dmin, int *next_block, int motion_x, int motion_y)
2325 {
2326     int score;
2327     uint8_t *dest_backup[3];
2328
2329     copy_context_before_encode(s, backup, type);
2330
2331     s->block= s->blocks[*next_block];
2332     s->pb= pb[*next_block];
2333     if(s->data_partitioning){
2334         s->pb2   = pb2   [*next_block];
2335         s->tex_pb= tex_pb[*next_block];
2336     }
2337
2338     if(*next_block){
2339         memcpy(dest_backup, s->dest, sizeof(s->dest));
2340         s->dest[0] = s->sc.rd_scratchpad;
2341         s->dest[1] = s->sc.rd_scratchpad + 16*s->linesize;
2342         s->dest[2] = s->sc.rd_scratchpad + 16*s->linesize + 8;
2343         assert(s->linesize >= 32); //FIXME
2344     }
2345
2346     encode_mb(s, motion_x, motion_y);
2347
2348     score= put_bits_count(&s->pb);
2349     if(s->data_partitioning){
2350         score+= put_bits_count(&s->pb2);
2351         score+= put_bits_count(&s->tex_pb);
2352     }
2353
2354     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2355         ff_mpv_decode_mb(s, s->block);
2356
2357         score *= s->lambda2;
2358         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2359     }
2360
2361     if(*next_block){
2362         memcpy(s->dest, dest_backup, sizeof(s->dest));
2363     }
2364
2365     if(score<*dmin){
2366         *dmin= score;
2367         *next_block^=1;
2368
2369         copy_context_after_encode(best, s, type);
2370     }
2371 }
2372
2373 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2374     uint32_t *sq = ff_square_tab + 256;
2375     int acc=0;
2376     int x,y;
2377
2378     if(w==16 && h==16)
2379         return s->mecc.sse[0](NULL, src1, src2, stride, 16);
2380     else if(w==8 && h==8)
2381         return s->mecc.sse[1](NULL, src1, src2, stride, 8);
2382
2383     for(y=0; y<h; y++){
2384         for(x=0; x<w; x++){
2385             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2386         }
2387     }
2388
2389     assert(acc>=0);
2390
2391     return acc;
2392 }
2393
2394 static int sse_mb(MpegEncContext *s){
2395     int w= 16;
2396     int h= 16;
2397
2398     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2399     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2400
2401     if(w==16 && h==16)
2402       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2403         return s->mecc.nsse[0](s, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2404                s->mecc.nsse[1](s, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2405                s->mecc.nsse[1](s, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2406       }else{
2407         return s->mecc.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2408                s->mecc.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2409                s->mecc.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2410       }
2411     else
2412         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2413                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2414                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2415 }
2416
2417 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2418     MpegEncContext *s= *(void**)arg;
2419
2420
2421     s->me.pre_pass=1;
2422     s->me.dia_size= s->avctx->pre_dia_size;
2423     s->first_slice_line=1;
2424     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2425         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2426             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2427         }
2428         s->first_slice_line=0;
2429     }
2430
2431     s->me.pre_pass=0;
2432
2433     return 0;
2434 }
2435
2436 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2437     MpegEncContext *s= *(void**)arg;
2438
2439     s->me.dia_size= s->avctx->dia_size;
2440     s->first_slice_line=1;
2441     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2442         s->mb_x=0; //for block init below
2443         ff_init_block_index(s);
2444         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2445             s->block_index[0]+=2;
2446             s->block_index[1]+=2;
2447             s->block_index[2]+=2;
2448             s->block_index[3]+=2;
2449
2450             /* compute motion vector & mb_type and store in context */
2451             if(s->pict_type==AV_PICTURE_TYPE_B)
2452                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2453             else
2454                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2455         }
2456         s->first_slice_line=0;
2457     }
2458     return 0;
2459 }
2460
2461 static int mb_var_thread(AVCodecContext *c, void *arg){
2462     MpegEncContext *s= *(void**)arg;
2463     int mb_x, mb_y;
2464
2465     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2466         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2467             int xx = mb_x * 16;
2468             int yy = mb_y * 16;
2469             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2470             int varc;
2471             int sum = s->mpvencdsp.pix_sum(pix, s->linesize);
2472
2473             varc = (s->mpvencdsp.pix_norm1(pix, s->linesize) -
2474                     (((unsigned) sum * sum) >> 8) + 500 + 128) >> 8;
2475
2476             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2477             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2478             s->me.mb_var_sum_temp    += varc;
2479         }
2480     }
2481     return 0;
2482 }
2483
2484 static void write_slice_end(MpegEncContext *s){
2485     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2486         if(s->partitioned_frame){
2487             ff_mpeg4_merge_partitions(s);
2488         }
2489
2490         ff_mpeg4_stuffing(&s->pb);
2491     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2492         ff_mjpeg_encode_stuffing(&s->pb);
2493     }
2494
2495     avpriv_align_put_bits(&s->pb);
2496     flush_put_bits(&s->pb);
2497
2498     if ((s->avctx->flags & CODEC_FLAG_PASS1) && !s->partitioned_frame)
2499         s->misc_bits+= get_bits_diff(s);
2500 }
2501
2502 static void write_mb_info(MpegEncContext *s)
2503 {
2504     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2505     int offset = put_bits_count(&s->pb);
2506     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2507     int gobn = s->mb_y / s->gob_index;
2508     int pred_x, pred_y;
2509     if (CONFIG_H263_ENCODER)
2510         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2511     bytestream_put_le32(&ptr, offset);
2512     bytestream_put_byte(&ptr, s->qscale);
2513     bytestream_put_byte(&ptr, gobn);
2514     bytestream_put_le16(&ptr, mba);
2515     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2516     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2517     /* 4MV not implemented */
2518     bytestream_put_byte(&ptr, 0); /* hmv2 */
2519     bytestream_put_byte(&ptr, 0); /* vmv2 */
2520 }
2521
2522 static void update_mb_info(MpegEncContext *s, int startcode)
2523 {
2524     if (!s->mb_info)
2525         return;
2526     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2527         s->mb_info_size += 12;
2528         s->prev_mb_info = s->last_mb_info;
2529     }
2530     if (startcode) {
2531         s->prev_mb_info = put_bits_count(&s->pb)/8;
2532         /* This might have incremented mb_info_size above, and we return without
2533          * actually writing any info into that slot yet. But in that case,
2534          * this will be called again at the start of the after writing the
2535          * start code, actually writing the mb info. */
2536         return;
2537     }
2538
2539     s->last_mb_info = put_bits_count(&s->pb)/8;
2540     if (!s->mb_info_size)
2541         s->mb_info_size += 12;
2542     write_mb_info(s);
2543 }
2544
2545 static int encode_thread(AVCodecContext *c, void *arg){
2546     MpegEncContext *s= *(void**)arg;
2547     int mb_x, mb_y, pdif = 0;
2548     int chr_h= 16>>s->chroma_y_shift;
2549     int i, j;
2550     MpegEncContext best_s = { 0 }, backup_s;
2551     uint8_t bit_buf[2][MAX_MB_BYTES];
2552     uint8_t bit_buf2[2][MAX_MB_BYTES];
2553     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2554     PutBitContext pb[2], pb2[2], tex_pb[2];
2555
2556     for(i=0; i<2; i++){
2557         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2558         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2559         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2560     }
2561
2562     s->last_bits= put_bits_count(&s->pb);
2563     s->mv_bits=0;
2564     s->misc_bits=0;
2565     s->i_tex_bits=0;
2566     s->p_tex_bits=0;
2567     s->i_count=0;
2568     s->f_count=0;
2569     s->b_count=0;
2570     s->skip_count=0;
2571
2572     for(i=0; i<3; i++){
2573         /* init last dc values */
2574         /* note: quant matrix value (8) is implied here */
2575         s->last_dc[i] = 128 << s->intra_dc_precision;
2576
2577         s->current_picture.f->error[i] = 0;
2578     }
2579     s->mb_skip_run = 0;
2580     memset(s->last_mv, 0, sizeof(s->last_mv));
2581
2582     s->last_mv_dir = 0;
2583
2584     switch(s->codec_id){
2585     case AV_CODEC_ID_H263:
2586     case AV_CODEC_ID_H263P:
2587     case AV_CODEC_ID_FLV1:
2588         if (CONFIG_H263_ENCODER)
2589             s->gob_index = H263_GOB_HEIGHT(s->height);
2590         break;
2591     case AV_CODEC_ID_MPEG4:
2592         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2593             ff_mpeg4_init_partitions(s);
2594         break;
2595     }
2596
2597     s->resync_mb_x=0;
2598     s->resync_mb_y=0;
2599     s->first_slice_line = 1;
2600     s->ptr_lastgob = s->pb.buf;
2601     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2602         s->mb_x=0;
2603         s->mb_y= mb_y;
2604
2605         ff_set_qscale(s, s->qscale);
2606         ff_init_block_index(s);
2607
2608         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2609             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2610             int mb_type= s->mb_type[xy];
2611 //            int d;
2612             int dmin= INT_MAX;
2613             int dir;
2614
2615             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2616                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2617                 return -1;
2618             }
2619             if(s->data_partitioning){
2620                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2621                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2622                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2623                     return -1;
2624                 }
2625             }
2626
2627             s->mb_x = mb_x;
2628             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2629             ff_update_block_index(s);
2630
2631             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
2632                 ff_h261_reorder_mb_index(s);
2633                 xy= s->mb_y*s->mb_stride + s->mb_x;
2634                 mb_type= s->mb_type[xy];
2635             }
2636
2637             /* write gob / video packet header  */
2638             if(s->rtp_mode){
2639                 int current_packet_size, is_gob_start;
2640
2641                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2642
2643                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2644
2645                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2646
2647                 switch(s->codec_id){
2648                 case AV_CODEC_ID_H263:
2649                 case AV_CODEC_ID_H263P:
2650                     if(!s->h263_slice_structured)
2651                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2652                     break;
2653                 case AV_CODEC_ID_MPEG2VIDEO:
2654                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2655                 case AV_CODEC_ID_MPEG1VIDEO:
2656                     if(s->mb_skip_run) is_gob_start=0;
2657                     break;
2658                 }
2659
2660                 if(is_gob_start){
2661                     if(s->start_mb_y != mb_y || mb_x!=0){
2662                         write_slice_end(s);
2663
2664                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
2665                             ff_mpeg4_init_partitions(s);
2666                         }
2667                     }
2668
2669                     assert((put_bits_count(&s->pb)&7) == 0);
2670                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2671
2672                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
2673                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2674                         int d = 100 / s->error_rate;
2675                         if(r % d == 0){
2676                             current_packet_size=0;
2677                             s->pb.buf_ptr= s->ptr_lastgob;
2678                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2679                         }
2680                     }
2681
2682                     if (s->avctx->rtp_callback){
2683                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2684                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2685                     }
2686                     update_mb_info(s, 1);
2687
2688                     switch(s->codec_id){
2689                     case AV_CODEC_ID_MPEG4:
2690                         if (CONFIG_MPEG4_ENCODER) {
2691                             ff_mpeg4_encode_video_packet_header(s);
2692                             ff_mpeg4_clean_buffers(s);
2693                         }
2694                     break;
2695                     case AV_CODEC_ID_MPEG1VIDEO:
2696                     case AV_CODEC_ID_MPEG2VIDEO:
2697                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2698                             ff_mpeg1_encode_slice_header(s);
2699                             ff_mpeg1_clean_buffers(s);
2700                         }
2701                     break;
2702                     case AV_CODEC_ID_H263:
2703                     case AV_CODEC_ID_H263P:
2704                         if (CONFIG_H263_ENCODER)
2705                             ff_h263_encode_gob_header(s, mb_y);
2706                     break;
2707                     }
2708
2709                     if (s->avctx->flags & CODEC_FLAG_PASS1) {
2710                         int bits= put_bits_count(&s->pb);
2711                         s->misc_bits+= bits - s->last_bits;
2712                         s->last_bits= bits;
2713                     }
2714
2715                     s->ptr_lastgob += current_packet_size;
2716                     s->first_slice_line=1;
2717                     s->resync_mb_x=mb_x;
2718                     s->resync_mb_y=mb_y;
2719                 }
2720             }
2721
2722             if(  (s->resync_mb_x   == s->mb_x)
2723                && s->resync_mb_y+1 == s->mb_y){
2724                 s->first_slice_line=0;
2725             }
2726
2727             s->mb_skipped=0;
2728             s->dquant=0; //only for QP_RD
2729
2730             update_mb_info(s, 0);
2731
2732             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
2733                 int next_block=0;
2734                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2735
2736                 copy_context_before_encode(&backup_s, s, -1);
2737                 backup_s.pb= s->pb;
2738                 best_s.data_partitioning= s->data_partitioning;
2739                 best_s.partitioned_frame= s->partitioned_frame;
2740                 if(s->data_partitioning){
2741                     backup_s.pb2= s->pb2;
2742                     backup_s.tex_pb= s->tex_pb;
2743                 }
2744
2745                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2746                     s->mv_dir = MV_DIR_FORWARD;
2747                     s->mv_type = MV_TYPE_16X16;
2748                     s->mb_intra= 0;
2749                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2750                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2751                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2752                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2753                 }
2754                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2755                     s->mv_dir = MV_DIR_FORWARD;
2756                     s->mv_type = MV_TYPE_FIELD;
2757                     s->mb_intra= 0;
2758                     for(i=0; i<2; i++){
2759                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2760                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2761                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2762                     }
2763                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2764                                  &dmin, &next_block, 0, 0);
2765                 }
2766                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2767                     s->mv_dir = MV_DIR_FORWARD;
2768                     s->mv_type = MV_TYPE_16X16;
2769                     s->mb_intra= 0;
2770                     s->mv[0][0][0] = 0;
2771                     s->mv[0][0][1] = 0;
2772                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2773                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2774                 }
2775                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2776                     s->mv_dir = MV_DIR_FORWARD;
2777                     s->mv_type = MV_TYPE_8X8;
2778                     s->mb_intra= 0;
2779                     for(i=0; i<4; i++){
2780                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2781                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2782                     }
2783                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2784                                  &dmin, &next_block, 0, 0);
2785                 }
2786                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2787                     s->mv_dir = MV_DIR_FORWARD;
2788                     s->mv_type = MV_TYPE_16X16;
2789                     s->mb_intra= 0;
2790                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2791                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2792                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2793                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2794                 }
2795                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2796                     s->mv_dir = MV_DIR_BACKWARD;
2797                     s->mv_type = MV_TYPE_16X16;
2798                     s->mb_intra= 0;
2799                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2800                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2801                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2802                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2803                 }
2804                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2805                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2806                     s->mv_type = MV_TYPE_16X16;
2807                     s->mb_intra= 0;
2808                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2809                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2810                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2811                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2812                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2813                                  &dmin, &next_block, 0, 0);
2814                 }
2815                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2816                     s->mv_dir = MV_DIR_FORWARD;
2817                     s->mv_type = MV_TYPE_FIELD;
2818                     s->mb_intra= 0;
2819                     for(i=0; i<2; i++){
2820                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2821                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2822                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2823                     }
2824                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2825                                  &dmin, &next_block, 0, 0);
2826                 }
2827                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2828                     s->mv_dir = MV_DIR_BACKWARD;
2829                     s->mv_type = MV_TYPE_FIELD;
2830                     s->mb_intra= 0;
2831                     for(i=0; i<2; i++){
2832                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2833                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2834                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2835                     }
2836                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2837                                  &dmin, &next_block, 0, 0);
2838                 }
2839                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2840                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2841                     s->mv_type = MV_TYPE_FIELD;
2842                     s->mb_intra= 0;
2843                     for(dir=0; dir<2; dir++){
2844                         for(i=0; i<2; i++){
2845                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2846                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2847                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2848                         }
2849                     }
2850                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2851                                  &dmin, &next_block, 0, 0);
2852                 }
2853                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2854                     s->mv_dir = 0;
2855                     s->mv_type = MV_TYPE_16X16;
2856                     s->mb_intra= 1;
2857                     s->mv[0][0][0] = 0;
2858                     s->mv[0][0][1] = 0;
2859                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2860                                  &dmin, &next_block, 0, 0);
2861                     if(s->h263_pred || s->h263_aic){
2862                         if(best_s.mb_intra)
2863                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2864                         else
2865                             ff_clean_intra_table_entries(s); //old mode?
2866                     }
2867                 }
2868
2869                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
2870                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2871                         const int last_qp= backup_s.qscale;
2872                         int qpi, qp, dc[6];
2873                         int16_t ac[6][16];
2874                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2875                         static const int dquant_tab[4]={-1,1,-2,2};
2876
2877                         assert(backup_s.dquant == 0);
2878
2879                         //FIXME intra
2880                         s->mv_dir= best_s.mv_dir;
2881                         s->mv_type = MV_TYPE_16X16;
2882                         s->mb_intra= best_s.mb_intra;
2883                         s->mv[0][0][0] = best_s.mv[0][0][0];
2884                         s->mv[0][0][1] = best_s.mv[0][0][1];
2885                         s->mv[1][0][0] = best_s.mv[1][0][0];
2886                         s->mv[1][0][1] = best_s.mv[1][0][1];
2887
2888                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
2889                         for(; qpi<4; qpi++){
2890                             int dquant= dquant_tab[qpi];
2891                             qp= last_qp + dquant;
2892                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2893                                 continue;
2894                             backup_s.dquant= dquant;
2895                             if(s->mb_intra && s->dc_val[0]){
2896                                 for(i=0; i<6; i++){
2897                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2898                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(int16_t)*16);
2899                                 }
2900                             }
2901
2902                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2903                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2904                             if(best_s.qscale != qp){
2905                                 if(s->mb_intra && s->dc_val[0]){
2906                                     for(i=0; i<6; i++){
2907                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2908                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(int16_t)*16);
2909                                     }
2910                                 }
2911                             }
2912                         }
2913                     }
2914                 }
2915                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2916                     int mx= s->b_direct_mv_table[xy][0];
2917                     int my= s->b_direct_mv_table[xy][1];
2918
2919                     backup_s.dquant = 0;
2920                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2921                     s->mb_intra= 0;
2922                     ff_mpeg4_set_direct_mv(s, mx, my);
2923                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2924                                  &dmin, &next_block, mx, my);
2925                 }
2926                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2927                     backup_s.dquant = 0;
2928                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2929                     s->mb_intra= 0;
2930                     ff_mpeg4_set_direct_mv(s, 0, 0);
2931                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2932                                  &dmin, &next_block, 0, 0);
2933                 }
2934                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
2935                     int coded=0;
2936                     for(i=0; i<6; i++)
2937                         coded |= s->block_last_index[i];
2938                     if(coded){
2939                         int mx,my;
2940                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2941                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
2942                             mx=my=0; //FIXME find the one we actually used
2943                             ff_mpeg4_set_direct_mv(s, mx, my);
2944                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2945                             mx= s->mv[1][0][0];
2946                             my= s->mv[1][0][1];
2947                         }else{
2948                             mx= s->mv[0][0][0];
2949                             my= s->mv[0][0][1];
2950                         }
2951
2952                         s->mv_dir= best_s.mv_dir;
2953                         s->mv_type = best_s.mv_type;
2954                         s->mb_intra= 0;
2955 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2956                         s->mv[0][0][1] = best_s.mv[0][0][1];
2957                         s->mv[1][0][0] = best_s.mv[1][0][0];
2958                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2959                         backup_s.dquant= 0;
2960                         s->skipdct=1;
2961                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2962                                         &dmin, &next_block, mx, my);
2963                         s->skipdct=0;
2964                     }
2965                 }
2966
2967                 s->current_picture.qscale_table[xy] = best_s.qscale;
2968
2969                 copy_context_after_encode(s, &best_s, -1);
2970
2971                 pb_bits_count= put_bits_count(&s->pb);
2972                 flush_put_bits(&s->pb);
2973                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2974                 s->pb= backup_s.pb;
2975
2976                 if(s->data_partitioning){
2977                     pb2_bits_count= put_bits_count(&s->pb2);
2978                     flush_put_bits(&s->pb2);
2979                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2980                     s->pb2= backup_s.pb2;
2981
2982                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2983                     flush_put_bits(&s->tex_pb);
2984                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2985                     s->tex_pb= backup_s.tex_pb;
2986                 }
2987                 s->last_bits= put_bits_count(&s->pb);
2988
2989                 if (CONFIG_H263_ENCODER &&
2990                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2991                     ff_h263_update_motion_val(s);
2992
2993                 if(next_block==0){ //FIXME 16 vs linesize16
2994                     s->hdsp.put_pixels_tab[0][0](s->dest[0], s->sc.rd_scratchpad                     , s->linesize  ,16);
2995                     s->hdsp.put_pixels_tab[1][0](s->dest[1], s->sc.rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2996                     s->hdsp.put_pixels_tab[1][0](s->dest[2], s->sc.rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2997                 }
2998
2999                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
3000                     ff_mpv_decode_mb(s, s->block);
3001             } else {
3002                 int motion_x = 0, motion_y = 0;
3003                 s->mv_type=MV_TYPE_16X16;
3004                 // only one MB-Type possible
3005
3006                 switch(mb_type){
3007                 case CANDIDATE_MB_TYPE_INTRA:
3008                     s->mv_dir = 0;
3009                     s->mb_intra= 1;
3010                     motion_x= s->mv[0][0][0] = 0;
3011                     motion_y= s->mv[0][0][1] = 0;
3012                     break;
3013                 case CANDIDATE_MB_TYPE_INTER:
3014                     s->mv_dir = MV_DIR_FORWARD;
3015                     s->mb_intra= 0;
3016                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
3017                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
3018                     break;
3019                 case CANDIDATE_MB_TYPE_INTER_I:
3020                     s->mv_dir = MV_DIR_FORWARD;
3021                     s->mv_type = MV_TYPE_FIELD;
3022                     s->mb_intra= 0;
3023                     for(i=0; i<2; i++){
3024                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3025                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3026                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3027                     }
3028                     break;
3029                 case CANDIDATE_MB_TYPE_INTER4V:
3030                     s->mv_dir = MV_DIR_FORWARD;
3031                     s->mv_type = MV_TYPE_8X8;
3032                     s->mb_intra= 0;
3033                     for(i=0; i<4; i++){
3034                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3035                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3036                     }
3037                     break;
3038                 case CANDIDATE_MB_TYPE_DIRECT:
3039                     if (CONFIG_MPEG4_ENCODER) {
3040                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3041                         s->mb_intra= 0;
3042                         motion_x=s->b_direct_mv_table[xy][0];
3043                         motion_y=s->b_direct_mv_table[xy][1];
3044                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
3045                     }
3046                     break;
3047                 case CANDIDATE_MB_TYPE_DIRECT0:
3048                     if (CONFIG_MPEG4_ENCODER) {
3049                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3050                         s->mb_intra= 0;
3051                         ff_mpeg4_set_direct_mv(s, 0, 0);
3052                     }
3053                     break;
3054                 case CANDIDATE_MB_TYPE_BIDIR:
3055                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3056                     s->mb_intra= 0;
3057                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3058                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3059                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3060                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3061                     break;
3062                 case CANDIDATE_MB_TYPE_BACKWARD:
3063                     s->mv_dir = MV_DIR_BACKWARD;
3064                     s->mb_intra= 0;
3065                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3066                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3067                     break;
3068                 case CANDIDATE_MB_TYPE_FORWARD:
3069                     s->mv_dir = MV_DIR_FORWARD;
3070                     s->mb_intra= 0;
3071                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3072                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3073                     break;
3074                 case CANDIDATE_MB_TYPE_FORWARD_I:
3075                     s->mv_dir = MV_DIR_FORWARD;
3076                     s->mv_type = MV_TYPE_FIELD;
3077                     s->mb_intra= 0;
3078                     for(i=0; i<2; i++){
3079                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3080                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3081                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3082                     }
3083                     break;
3084                 case CANDIDATE_MB_TYPE_BACKWARD_I:
3085                     s->mv_dir = MV_DIR_BACKWARD;
3086                     s->mv_type = MV_TYPE_FIELD;
3087                     s->mb_intra= 0;
3088                     for(i=0; i<2; i++){
3089                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3090                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3091                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3092                     }
3093                     break;
3094                 case CANDIDATE_MB_TYPE_BIDIR_I:
3095                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3096                     s->mv_type = MV_TYPE_FIELD;
3097                     s->mb_intra= 0;
3098                     for(dir=0; dir<2; dir++){
3099                         for(i=0; i<2; i++){
3100                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3101                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3102                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3103                         }
3104                     }
3105                     break;
3106                 default:
3107                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
3108                 }
3109
3110                 encode_mb(s, motion_x, motion_y);
3111
3112                 // RAL: Update last macroblock type
3113                 s->last_mv_dir = s->mv_dir;
3114
3115                 if (CONFIG_H263_ENCODER &&
3116                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3117                     ff_h263_update_motion_val(s);
3118
3119                 ff_mpv_decode_mb(s, s->block);
3120             }
3121
3122             /* clean the MV table in IPS frames for direct mode in B frames */
3123             if(s->mb_intra /* && I,P,S_TYPE */){
3124                 s->p_mv_table[xy][0]=0;
3125                 s->p_mv_table[xy][1]=0;
3126             }
3127
3128             if (s->avctx->flags & CODEC_FLAG_PSNR) {
3129                 int w= 16;
3130                 int h= 16;
3131
3132                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
3133                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
3134
3135                 s->current_picture.f->error[0] += sse(
3136                     s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
3137                     s->dest[0], w, h, s->linesize);
3138                 s->current_picture.f->error[1] += sse(
3139                     s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3140                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3141                 s->current_picture.f->error[2] += sse(
3142                     s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3143                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3144             }
3145             if(s->loop_filter){
3146                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
3147                     ff_h263_loop_filter(s);
3148             }
3149             ff_dlog(s->avctx, "MB %d %d bits\n",
3150                     s->mb_x + s->mb_y * s->mb_stride, put_bits_count(&s->pb));
3151         }
3152     }
3153
3154     //not beautiful here but we must write it before flushing so it has to be here
3155     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
3156         ff_msmpeg4_encode_ext_header(s);
3157
3158     write_slice_end(s);
3159
3160     /* Send the last GOB if RTP */
3161     if (s->avctx->rtp_callback) {
3162         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
3163         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
3164         /* Call the RTP callback to send the last GOB */
3165         emms_c();
3166         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
3167     }
3168
3169     return 0;
3170 }
3171
3172 #define MERGE(field) dst->field += src->field; src->field=0
3173 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
3174     MERGE(me.scene_change_score);
3175     MERGE(me.mc_mb_var_sum_temp);
3176     MERGE(me.mb_var_sum_temp);
3177 }
3178
3179 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
3180     int i;
3181
3182     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
3183     MERGE(dct_count[1]);
3184     MERGE(mv_bits);
3185     MERGE(i_tex_bits);
3186     MERGE(p_tex_bits);
3187     MERGE(i_count);
3188     MERGE(f_count);
3189     MERGE(b_count);
3190     MERGE(skip_count);
3191     MERGE(misc_bits);
3192     MERGE(er.error_count);
3193     MERGE(padding_bug_score);
3194     MERGE(current_picture.f->error[0]);
3195     MERGE(current_picture.f->error[1]);
3196     MERGE(current_picture.f->error[2]);
3197
3198     if(dst->avctx->noise_reduction){
3199         for(i=0; i<64; i++){
3200             MERGE(dct_error_sum[0][i]);
3201             MERGE(dct_error_sum[1][i]);
3202         }
3203     }
3204
3205     assert(put_bits_count(&src->pb) % 8 ==0);
3206     assert(put_bits_count(&dst->pb) % 8 ==0);
3207     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3208     flush_put_bits(&dst->pb);
3209 }
3210
3211 static int estimate_qp(MpegEncContext *s, int dry_run){
3212     if (s->next_lambda){
3213         s->current_picture_ptr->f->quality =
3214         s->current_picture.f->quality = s->next_lambda;
3215         if(!dry_run) s->next_lambda= 0;
3216     } else if (!s->fixed_qscale) {
3217         s->current_picture_ptr->f->quality =
3218         s->current_picture.f->quality = ff_rate_estimate_qscale(s, dry_run);
3219         if (s->current_picture.f->quality < 0)
3220             return -1;
3221     }
3222
3223     if(s->adaptive_quant){
3224         switch(s->codec_id){
3225         case AV_CODEC_ID_MPEG4:
3226             if (CONFIG_MPEG4_ENCODER)
3227                 ff_clean_mpeg4_qscales(s);
3228             break;
3229         case AV_CODEC_ID_H263:
3230         case AV_CODEC_ID_H263P:
3231         case AV_CODEC_ID_FLV1:
3232             if (CONFIG_H263_ENCODER)
3233                 ff_clean_h263_qscales(s);
3234             break;
3235         default:
3236             ff_init_qscale_tab(s);
3237         }
3238
3239         s->lambda= s->lambda_table[0];
3240         //FIXME broken
3241     }else
3242         s->lambda = s->current_picture.f->quality;
3243     update_qscale(s);
3244     return 0;
3245 }
3246
3247 /* must be called before writing the header */
3248 static void set_frame_distances(MpegEncContext * s){
3249     assert(s->current_picture_ptr->f->pts != AV_NOPTS_VALUE);
3250     s->time = s->current_picture_ptr->f->pts * s->avctx->time_base.num;
3251
3252     if(s->pict_type==AV_PICTURE_TYPE_B){
3253         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3254         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3255     }else{
3256         s->pp_time= s->time - s->last_non_b_time;
3257         s->last_non_b_time= s->time;
3258         assert(s->picture_number==0 || s->pp_time > 0);
3259     }
3260 }
3261
3262 static int encode_picture(MpegEncContext *s, int picture_number)
3263 {
3264     int i, ret;
3265     int bits;
3266     int context_count = s->slice_context_count;
3267
3268     s->picture_number = picture_number;
3269
3270     /* Reset the average MB variance */
3271     s->me.mb_var_sum_temp    =
3272     s->me.mc_mb_var_sum_temp = 0;
3273
3274     /* we need to initialize some time vars before we can encode b-frames */
3275     // RAL: Condition added for MPEG1VIDEO
3276     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3277         set_frame_distances(s);
3278     if(CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4)
3279         ff_set_mpeg4_time(s);
3280
3281     s->me.scene_change_score=0;
3282
3283 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3284
3285     if(s->pict_type==AV_PICTURE_TYPE_I){
3286         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3287         else                        s->no_rounding=0;
3288     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3289         if(s->flipflop_rounding || s->codec_id == AV_CODEC_ID_H263P || s->codec_id == AV_CODEC_ID_MPEG4)
3290             s->no_rounding ^= 1;
3291     }
3292
3293     if (s->avctx->flags & CODEC_FLAG_PASS2) {
3294         if (estimate_qp(s,1) < 0)
3295             return -1;
3296         ff_get_2pass_fcode(s);
3297     } else if (!(s->avctx->flags & CODEC_FLAG_QSCALE)) {
3298         if(s->pict_type==AV_PICTURE_TYPE_B)
3299             s->lambda= s->last_lambda_for[s->pict_type];
3300         else
3301             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3302         update_qscale(s);
3303     }
3304
3305     s->mb_intra=0; //for the rate distortion & bit compare functions
3306     for(i=1; i<context_count; i++){
3307         ret = ff_update_duplicate_context(s->thread_context[i], s);
3308         if (ret < 0)
3309             return ret;
3310     }
3311
3312     if(ff_init_me(s)<0)
3313         return -1;
3314
3315     /* Estimate motion for every MB */
3316     if(s->pict_type != AV_PICTURE_TYPE_I){
3317         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
3318         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
3319         if (s->pict_type != AV_PICTURE_TYPE_B) {
3320             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
3321                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3322             }
3323         }
3324
3325         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3326     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3327         /* I-Frame */
3328         for(i=0; i<s->mb_stride*s->mb_height; i++)
3329             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3330
3331         if(!s->fixed_qscale){
3332             /* finding spatial complexity for I-frame rate control */
3333             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3334         }
3335     }
3336     for(i=1; i<context_count; i++){
3337         merge_context_after_me(s, s->thread_context[i]);
3338     }
3339     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3340     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3341     emms_c();
3342
3343     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
3344         s->pict_type= AV_PICTURE_TYPE_I;
3345         for(i=0; i<s->mb_stride*s->mb_height; i++)
3346             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3347         ff_dlog(s, "Scene change detected, encoding as I Frame %d %d\n",
3348                 s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3349     }
3350
3351     if(!s->umvplus){
3352         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3353             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3354
3355             if (s->avctx->flags & CODEC_FLAG_INTERLACED_ME) {
3356                 int a,b;
3357                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3358                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3359                 s->f_code= FFMAX3(s->f_code, a, b);
3360             }
3361
3362             ff_fix_long_p_mvs(s);
3363             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3364             if (s->avctx->flags & CODEC_FLAG_INTERLACED_ME) {
3365                 int j;
3366                 for(i=0; i<2; i++){
3367                     for(j=0; j<2; j++)
3368                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3369                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3370                 }
3371             }
3372         }
3373
3374         if(s->pict_type==AV_PICTURE_TYPE_B){
3375             int a, b;
3376
3377             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3378             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3379             s->f_code = FFMAX(a, b);
3380
3381             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3382             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3383             s->b_code = FFMAX(a, b);
3384
3385             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3386             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3387             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3388             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3389             if (s->avctx->flags & CODEC_FLAG_INTERLACED_ME) {
3390                 int dir, j;
3391                 for(dir=0; dir<2; dir++){
3392                     for(i=0; i<2; i++){
3393                         for(j=0; j<2; j++){
3394                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3395                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3396                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3397                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3398                         }
3399                     }
3400                 }
3401             }
3402         }
3403     }
3404
3405     if (estimate_qp(s, 0) < 0)
3406         return -1;
3407
3408     if (s->qscale < 3 && s->max_qcoeff <= 128 &&
3409         s->pict_type == AV_PICTURE_TYPE_I &&
3410         !(s->avctx->flags & CODEC_FLAG_QSCALE))
3411         s->qscale= 3; //reduce clipping problems
3412
3413     if (s->out_format == FMT_MJPEG) {
3414         /* for mjpeg, we do include qscale in the matrix */
3415         for(i=1;i<64;i++){
3416             int j = s->idsp.idct_permutation[i];
3417
3418             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
3419         }
3420         s->y_dc_scale_table=
3421         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3422         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3423         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
3424                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3425         s->qscale= 8;
3426     }
3427
3428     //FIXME var duplication
3429     s->current_picture_ptr->f->key_frame =
3430     s->current_picture.f->key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3431     s->current_picture_ptr->f->pict_type =
3432     s->current_picture.f->pict_type = s->pict_type;
3433
3434     if (s->current_picture.f->key_frame)
3435         s->picture_in_gop_number=0;
3436
3437     s->last_bits= put_bits_count(&s->pb);
3438     switch(s->out_format) {
3439     case FMT_MJPEG:
3440         if (CONFIG_MJPEG_ENCODER)
3441             ff_mjpeg_encode_picture_header(s->avctx, &s->pb, &s->intra_scantable,
3442                                            s->intra_matrix);
3443         break;
3444     case FMT_H261:
3445         if (CONFIG_H261_ENCODER)
3446             ff_h261_encode_picture_header(s, picture_number);
3447         break;
3448     case FMT_H263:
3449         if (CONFIG_WMV2_ENCODER && s->codec_id == AV_CODEC_ID_WMV2)
3450             ff_wmv2_encode_picture_header(s, picture_number);
3451         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3452             ff_msmpeg4_encode_picture_header(s, picture_number);
3453         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
3454             ff_mpeg4_encode_picture_header(s, picture_number);
3455         else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10) {
3456             ret = ff_rv10_encode_picture_header(s, picture_number);
3457             if (ret < 0)
3458                 return ret;
3459         }
3460         else if (CONFIG_RV20_ENCODER && s->codec_id == AV_CODEC_ID_RV20)
3461             ff_rv20_encode_picture_header(s, picture_number);
3462         else if (CONFIG_FLV_ENCODER && s->codec_id == AV_CODEC_ID_FLV1)
3463             ff_flv_encode_picture_header(s, picture_number);
3464         else if (CONFIG_H263_ENCODER)
3465             ff_h263_encode_picture_header(s, picture_number);
3466         break;
3467     case FMT_MPEG1:
3468         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
3469             ff_mpeg1_encode_picture_header(s, picture_number);
3470         break;
3471     default:
3472         assert(0);
3473     }
3474     bits= put_bits_count(&s->pb);
3475     s->header_bits= bits - s->last_bits;
3476
3477     for(i=1; i<context_count; i++){
3478         update_duplicate_context_after_me(s->thread_context[i], s);
3479     }
3480     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3481     for(i=1; i<context_count; i++){
3482         merge_context_after_encode(s, s->thread_context[i]);
3483     }
3484     emms_c();
3485     return 0;
3486 }
3487
3488 static void denoise_dct_c(MpegEncContext *s, int16_t *block){
3489     const int intra= s->mb_intra;
3490     int i;
3491
3492     s->dct_count[intra]++;
3493
3494     for(i=0; i<64; i++){
3495         int level= block[i];
3496
3497         if(level){
3498             if(level>0){
3499                 s->dct_error_sum[intra][i] += level;
3500                 level -= s->dct_offset[intra][i];
3501                 if(level<0) level=0;
3502             }else{
3503                 s->dct_error_sum[intra][i] -= level;
3504                 level += s->dct_offset[intra][i];
3505                 if(level>0) level=0;
3506             }
3507             block[i]= level;
3508         }
3509     }
3510 }
3511
3512 static int dct_quantize_trellis_c(MpegEncContext *s,
3513                                   int16_t *block, int n,
3514                                   int qscale, int *overflow){
3515     const int *qmat;
3516     const uint8_t *scantable= s->intra_scantable.scantable;
3517     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3518     int max=0;
3519     unsigned int threshold1, threshold2;
3520     int bias=0;
3521     int run_tab[65];
3522     int level_tab[65];
3523     int score_tab[65];
3524     int survivor[65];
3525     int survivor_count;
3526     int last_run=0;
3527     int last_level=0;
3528     int last_score= 0;
3529     int last_i;
3530     int coeff[2][64];
3531     int coeff_count[64];
3532     int qmul, qadd, start_i, last_non_zero, i, dc;
3533     const int esc_length= s->ac_esc_length;
3534     uint8_t * length;
3535     uint8_t * last_length;
3536     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3537
3538     s->fdsp.fdct(block);
3539
3540     if(s->dct_error_sum)
3541         s->denoise_dct(s, block);
3542     qmul= qscale*16;
3543     qadd= ((qscale-1)|1)*8;
3544
3545     if (s->mb_intra) {
3546         int q;
3547         if (!s->h263_aic) {
3548             if (n < 4)
3549                 q = s->y_dc_scale;
3550             else
3551                 q = s->c_dc_scale;
3552             q = q << 3;
3553         } else{
3554             /* For AIC we skip quant/dequant of INTRADC */
3555             q = 1 << 3;
3556             qadd=0;
3557         }
3558
3559         /* note: block[0] is assumed to be positive */
3560         block[0] = (block[0] + (q >> 1)) / q;
3561         start_i = 1;
3562         last_non_zero = 0;
3563         qmat = s->q_intra_matrix[qscale];
3564         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3565             bias= 1<<(QMAT_SHIFT-1);
3566         length     = s->intra_ac_vlc_length;
3567         last_length= s->intra_ac_vlc_last_length;
3568     } else {
3569         start_i = 0;
3570         last_non_zero = -1;
3571         qmat = s->q_inter_matrix[qscale];
3572         length     = s->inter_ac_vlc_length;
3573         last_length= s->inter_ac_vlc_last_length;
3574     }
3575     last_i= start_i;
3576
3577     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3578     threshold2= (threshold1<<1);
3579
3580     for(i=63; i>=start_i; i--) {
3581         const int j = scantable[i];
3582         int level = block[j] * qmat[j];
3583
3584         if(((unsigned)(level+threshold1))>threshold2){
3585             last_non_zero = i;
3586             break;
3587         }
3588     }
3589
3590     for(i=start_i; i<=last_non_zero; i++) {
3591         const int j = scantable[i];
3592         int level = block[j] * qmat[j];
3593
3594 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3595 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3596         if(((unsigned)(level+threshold1))>threshold2){
3597             if(level>0){
3598                 level= (bias + level)>>QMAT_SHIFT;
3599                 coeff[0][i]= level;
3600                 coeff[1][i]= level-1;
3601 //                coeff[2][k]= level-2;
3602             }else{
3603                 level= (bias - level)>>QMAT_SHIFT;
3604                 coeff[0][i]= -level;
3605                 coeff[1][i]= -level+1;
3606 //                coeff[2][k]= -level+2;
3607             }
3608             coeff_count[i]= FFMIN(level, 2);
3609             assert(coeff_count[i]);
3610             max |=level;
3611         }else{
3612             coeff[0][i]= (level>>31)|1;
3613             coeff_count[i]= 1;
3614         }
3615     }
3616
3617     *overflow= s->max_qcoeff < max; //overflow might have happened
3618
3619     if(last_non_zero < start_i){
3620         memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
3621         return last_non_zero;
3622     }
3623
3624     score_tab[start_i]= 0;
3625     survivor[0]= start_i;
3626     survivor_count= 1;
3627
3628     for(i=start_i; i<=last_non_zero; i++){
3629         int level_index, j, zero_distortion;
3630         int dct_coeff= FFABS(block[ scantable[i] ]);
3631         int best_score=256*256*256*120;
3632
3633         if (s->fdsp.fdct == ff_fdct_ifast)
3634             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3635         zero_distortion= dct_coeff*dct_coeff;
3636
3637         for(level_index=0; level_index < coeff_count[i]; level_index++){
3638             int distortion;
3639             int level= coeff[level_index][i];
3640             const int alevel= FFABS(level);
3641             int unquant_coeff;
3642
3643             assert(level);
3644
3645             if(s->out_format == FMT_H263){
3646                 unquant_coeff= alevel*qmul + qadd;
3647             }else{ //MPEG1
3648                 j = s->idsp.idct_permutation[scantable[i]]; // FIXME: optimize
3649                 if(s->mb_intra){
3650                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3651                         unquant_coeff =   (unquant_coeff - 1) | 1;
3652                 }else{
3653                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3654                         unquant_coeff =   (unquant_coeff - 1) | 1;
3655                 }
3656                 unquant_coeff<<= 3;
3657             }
3658
3659             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3660             level+=64;
3661             if((level&(~127)) == 0){
3662                 for(j=survivor_count-1; j>=0; j--){
3663                     int run= i - survivor[j];
3664                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3665                     score += score_tab[i-run];
3666
3667                     if(score < best_score){
3668                         best_score= score;
3669                         run_tab[i+1]= run;
3670                         level_tab[i+1]= level-64;
3671                     }
3672                 }
3673
3674                 if(s->out_format == FMT_H263){
3675                     for(j=survivor_count-1; j>=0; j--){
3676                         int run= i - survivor[j];
3677                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3678                         score += score_tab[i-run];
3679                         if(score < last_score){
3680                             last_score= score;
3681                             last_run= run;
3682                             last_level= level-64;
3683                             last_i= i+1;
3684                         }
3685                     }
3686                 }
3687             }else{
3688                 distortion += esc_length*lambda;
3689                 for(j=survivor_count-1; j>=0; j--){
3690                     int run= i - survivor[j];
3691                     int score= distortion + score_tab[i-run];
3692
3693                     if(score < best_score){
3694                         best_score= score;
3695                         run_tab[i+1]= run;
3696                         level_tab[i+1]= level-64;
3697                     }
3698                 }
3699
3700                 if(s->out_format == FMT_H263){
3701                   for(j=survivor_count-1; j>=0; j--){
3702                         int run= i - survivor[j];
3703                         int score= distortion + score_tab[i-run];
3704                         if(score < last_score){
3705                             last_score= score;
3706                             last_run= run;
3707                             last_level= level-64;
3708                             last_i= i+1;
3709                         }
3710                     }
3711                 }
3712             }
3713         }
3714
3715         score_tab[i+1]= best_score;
3716
3717         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
3718         if(last_non_zero <= 27){
3719             for(; survivor_count; survivor_count--){
3720                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3721                     break;
3722             }
3723         }else{
3724             for(; survivor_count; survivor_count--){
3725                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3726                     break;
3727             }
3728         }
3729
3730         survivor[ survivor_count++ ]= i+1;
3731     }
3732
3733     if(s->out_format != FMT_H263){
3734         last_score= 256*256*256*120;
3735         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3736             int score= score_tab[i];
3737             if(i) score += lambda*2; //FIXME exacter?
3738
3739             if(score < last_score){
3740                 last_score= score;
3741                 last_i= i;
3742                 last_level= level_tab[i];
3743                 last_run= run_tab[i];
3744             }
3745         }
3746     }
3747
3748     s->coded_score[n] = last_score;
3749
3750     dc= FFABS(block[0]);
3751     last_non_zero= last_i - 1;
3752     memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
3753
3754     if(last_non_zero < start_i)
3755         return last_non_zero;
3756
3757     if(last_non_zero == 0 && start_i == 0){
3758         int best_level= 0;
3759         int best_score= dc * dc;
3760
3761         for(i=0; i<coeff_count[0]; i++){
3762             int level= coeff[i][0];
3763             int alevel= FFABS(level);
3764             int unquant_coeff, score, distortion;
3765
3766             if(s->out_format == FMT_H263){
3767                     unquant_coeff= (alevel*qmul + qadd)>>3;
3768             }else{ //MPEG1
3769                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3770                     unquant_coeff =   (unquant_coeff - 1) | 1;
3771             }
3772             unquant_coeff = (unquant_coeff + 4) >> 3;
3773             unquant_coeff<<= 3 + 3;
3774
3775             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3776             level+=64;
3777             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3778             else                    score= distortion + esc_length*lambda;
3779
3780             if(score < best_score){
3781                 best_score= score;
3782                 best_level= level - 64;
3783             }
3784         }
3785         block[0]= best_level;
3786         s->coded_score[n] = best_score - dc*dc;
3787         if(best_level == 0) return -1;
3788         else                return last_non_zero;
3789     }
3790
3791     i= last_i;
3792     assert(last_level);
3793
3794     block[ perm_scantable[last_non_zero] ]= last_level;
3795     i -= last_run + 1;
3796
3797     for(; i>start_i; i -= run_tab[i] + 1){
3798         block[ perm_scantable[i-1] ]= level_tab[i];
3799     }
3800
3801     return last_non_zero;
3802 }
3803
3804 //#define REFINE_STATS 1
3805 static int16_t basis[64][64];
3806
3807 static void build_basis(uint8_t *perm){
3808     int i, j, x, y;
3809     emms_c();
3810     for(i=0; i<8; i++){
3811         for(j=0; j<8; j++){
3812             for(y=0; y<8; y++){
3813                 for(x=0; x<8; x++){
3814                     double s= 0.25*(1<<BASIS_SHIFT);
3815                     int index= 8*i + j;
3816                     int perm_index= perm[index];
3817                     if(i==0) s*= sqrt(0.5);
3818                     if(j==0) s*= sqrt(0.5);
3819                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3820                 }
3821             }
3822         }
3823     }
3824 }
3825
3826 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3827                         int16_t *block, int16_t *weight, int16_t *orig,
3828                         int n, int qscale){
3829     int16_t rem[64];
3830     LOCAL_ALIGNED_16(int16_t, d1, [64]);
3831     const uint8_t *scantable= s->intra_scantable.scantable;
3832     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3833 //    unsigned int threshold1, threshold2;
3834 //    int bias=0;
3835     int run_tab[65];
3836     int prev_run=0;
3837     int prev_level=0;
3838     int qmul, qadd, start_i, last_non_zero, i, dc;
3839     uint8_t * length;
3840     uint8_t * last_length;
3841     int lambda;
3842     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3843 #ifdef REFINE_STATS
3844 static int count=0;
3845 static int after_last=0;
3846 static int to_zero=0;
3847 static int from_zero=0;
3848 static int raise=0;
3849 static int lower=0;
3850 static int messed_sign=0;