ba17b2c6cebd62ff33eb5c49067a7575e43efd43
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of FFmpeg.
9  *
10  * FFmpeg is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * FFmpeg is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with FFmpeg; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include <stdint.h>
31
32 #include "libavutil/internal.h"
33 #include "libavutil/intmath.h"
34 #include "libavutil/mathematics.h"
35 #include "libavutil/pixdesc.h"
36 #include "libavutil/opt.h"
37 #include "libavutil/timer.h"
38 #include "avcodec.h"
39 #include "dct.h"
40 #include "dsputil.h"
41 #include "mpeg12.h"
42 #include "mpegvideo.h"
43 #include "h261.h"
44 #include "h263.h"
45 #include "mathops.h"
46 #include "mpegutils.h"
47 #include "mjpegenc.h"
48 #include "msmpeg4.h"
49 #include "qpeldsp.h"
50 #include "faandct.h"
51 #include "thread.h"
52 #include "aandcttab.h"
53 #include "flv.h"
54 #include "mpeg4video.h"
55 #include "internal.h"
56 #include "bytestream.h"
57 #include <limits.h>
58 #include "sp5x.h"
59
60 static int encode_picture(MpegEncContext *s, int picture_number);
61 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
62 static int sse_mb(MpegEncContext *s);
63 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
64 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
65
66 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
67 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
68
69 const AVOption ff_mpv_generic_options[] = {
70     FF_MPV_COMMON_OPTS
71     { NULL },
72 };
73
74 void ff_convert_matrix(DSPContext *dsp, int (*qmat)[64],
75                        uint16_t (*qmat16)[2][64],
76                        const uint16_t *quant_matrix,
77                        int bias, int qmin, int qmax, int intra)
78 {
79     int qscale;
80     int shift = 0;
81
82     for (qscale = qmin; qscale <= qmax; qscale++) {
83         int i;
84         if (dsp->fdct == ff_jpeg_fdct_islow_8 ||
85             dsp->fdct == ff_jpeg_fdct_islow_10 ||
86             dsp->fdct == ff_faandct) {
87             for (i = 0; i < 64; i++) {
88                 const int j = dsp->idct_permutation[i];
89                 /* 16 <= qscale * quant_matrix[i] <= 7905
90                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
91                  *             19952 <=              x  <= 249205026
92                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
93                  *           3444240 >= (1 << 36) / (x) >= 275 */
94
95                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
96                                         (qscale * quant_matrix[j]));
97             }
98         } else if (dsp->fdct == ff_fdct_ifast) {
99             for (i = 0; i < 64; i++) {
100                 const int j = dsp->idct_permutation[i];
101                 /* 16 <= qscale * quant_matrix[i] <= 7905
102                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
103                  *             19952 <=              x  <= 249205026
104                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
105                  *           3444240 >= (1 << 36) / (x) >= 275 */
106
107                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) /
108                                         (ff_aanscales[i] * (int64_t)qscale * quant_matrix[j]));
109             }
110         } else {
111             for (i = 0; i < 64; i++) {
112                 const int j = dsp->idct_permutation[i];
113                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
114                  * Assume x = qscale * quant_matrix[i]
115                  * So             16 <=              x  <= 7905
116                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
117                  * so          32768 >= (1 << 19) / (x) >= 67 */
118                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
119                                         (qscale * quant_matrix[j]));
120                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
121                 //                    (qscale * quant_matrix[i]);
122                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) /
123                                        (qscale * quant_matrix[j]);
124
125                 if (qmat16[qscale][0][i] == 0 ||
126                     qmat16[qscale][0][i] == 128 * 256)
127                     qmat16[qscale][0][i] = 128 * 256 - 1;
128                 qmat16[qscale][1][i] =
129                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
130                                 qmat16[qscale][0][i]);
131             }
132         }
133
134         for (i = intra; i < 64; i++) {
135             int64_t max = 8191;
136             if (dsp->fdct == ff_fdct_ifast) {
137                 max = (8191LL * ff_aanscales[i]) >> 14;
138             }
139             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
140                 shift++;
141             }
142         }
143     }
144     if (shift) {
145         av_log(NULL, AV_LOG_INFO,
146                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
147                QMAT_SHIFT - shift);
148     }
149 }
150
151 static inline void update_qscale(MpegEncContext *s)
152 {
153     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
154                 (FF_LAMBDA_SHIFT + 7);
155     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
156
157     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
158                  FF_LAMBDA_SHIFT;
159 }
160
161 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
162 {
163     int i;
164
165     if (matrix) {
166         put_bits(pb, 1, 1);
167         for (i = 0; i < 64; i++) {
168             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
169         }
170     } else
171         put_bits(pb, 1, 0);
172 }
173
174 /**
175  * init s->current_picture.qscale_table from s->lambda_table
176  */
177 void ff_init_qscale_tab(MpegEncContext *s)
178 {
179     int8_t * const qscale_table = s->current_picture.qscale_table;
180     int i;
181
182     for (i = 0; i < s->mb_num; i++) {
183         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
184         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
185         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
186                                                   s->avctx->qmax);
187     }
188 }
189
190 static void update_duplicate_context_after_me(MpegEncContext *dst,
191                                               MpegEncContext *src)
192 {
193 #define COPY(a) dst->a= src->a
194     COPY(pict_type);
195     COPY(current_picture);
196     COPY(f_code);
197     COPY(b_code);
198     COPY(qscale);
199     COPY(lambda);
200     COPY(lambda2);
201     COPY(picture_in_gop_number);
202     COPY(gop_picture_number);
203     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
204     COPY(progressive_frame);    // FIXME don't set in encode_header
205     COPY(partitioned_frame);    // FIXME don't set in encode_header
206 #undef COPY
207 }
208
209 /**
210  * Set the given MpegEncContext to defaults for encoding.
211  * the changed fields will not depend upon the prior state of the MpegEncContext.
212  */
213 static void MPV_encode_defaults(MpegEncContext *s)
214 {
215     int i;
216     ff_MPV_common_defaults(s);
217
218     for (i = -16; i < 16; i++) {
219         default_fcode_tab[i + MAX_MV] = 1;
220     }
221     s->me.mv_penalty = default_mv_penalty;
222     s->fcode_tab     = default_fcode_tab;
223
224     s->input_picture_number  = 0;
225     s->picture_in_gop_number = 0;
226 }
227
228 av_cold int ff_dct_encode_init(MpegEncContext *s) {
229     if (ARCH_X86)
230         ff_dct_encode_init_x86(s);
231
232     if (CONFIG_H263_ENCODER)
233         ff_h263dsp_init(&s->h263dsp);
234     if (!s->dct_quantize)
235         s->dct_quantize = ff_dct_quantize_c;
236     if (!s->denoise_dct)
237         s->denoise_dct  = denoise_dct_c;
238     s->fast_dct_quantize = s->dct_quantize;
239     if (s->avctx->trellis)
240         s->dct_quantize  = dct_quantize_trellis_c;
241
242     return 0;
243 }
244
245 /* init video encoder */
246 av_cold int ff_MPV_encode_init(AVCodecContext *avctx)
247 {
248     MpegEncContext *s = avctx->priv_data;
249     int i, ret, format_supported;
250
251     MPV_encode_defaults(s);
252
253     switch (avctx->codec_id) {
254     case AV_CODEC_ID_MPEG2VIDEO:
255         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
256             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
257             av_log(avctx, AV_LOG_ERROR,
258                    "only YUV420 and YUV422 are supported\n");
259             return -1;
260         }
261         break;
262     case AV_CODEC_ID_MJPEG:
263     case AV_CODEC_ID_AMV:
264         format_supported = 0;
265         /* JPEG color space */
266         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
267             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
268             avctx->pix_fmt == AV_PIX_FMT_YUVJ444P ||
269             (avctx->color_range == AVCOL_RANGE_JPEG &&
270              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
271               avctx->pix_fmt == AV_PIX_FMT_YUV422P ||
272               avctx->pix_fmt == AV_PIX_FMT_YUV444P)))
273             format_supported = 1;
274         /* MPEG color space */
275         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
276                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
277                   avctx->pix_fmt == AV_PIX_FMT_YUV422P ||
278                   avctx->pix_fmt == AV_PIX_FMT_YUV444P))
279             format_supported = 1;
280
281         if (!format_supported) {
282             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
283             return -1;
284         }
285         break;
286     default:
287         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
288             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
289             return -1;
290         }
291     }
292
293     switch (avctx->pix_fmt) {
294     case AV_PIX_FMT_YUVJ444P:
295     case AV_PIX_FMT_YUV444P:
296         s->chroma_format = CHROMA_444;
297         break;
298     case AV_PIX_FMT_YUVJ422P:
299     case AV_PIX_FMT_YUV422P:
300         s->chroma_format = CHROMA_422;
301         break;
302     case AV_PIX_FMT_YUVJ420P:
303     case AV_PIX_FMT_YUV420P:
304     default:
305         s->chroma_format = CHROMA_420;
306         break;
307     }
308
309     s->bit_rate = avctx->bit_rate;
310     s->width    = avctx->width;
311     s->height   = avctx->height;
312     if (avctx->gop_size > 600 &&
313         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
314         av_log(avctx, AV_LOG_WARNING,
315                "keyframe interval too large!, reducing it from %d to %d\n",
316                avctx->gop_size, 600);
317         avctx->gop_size = 600;
318     }
319     s->gop_size     = avctx->gop_size;
320     s->avctx        = avctx;
321     s->flags        = avctx->flags;
322     s->flags2       = avctx->flags2;
323     if (avctx->max_b_frames > MAX_B_FRAMES) {
324         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
325                "is %d.\n", MAX_B_FRAMES);
326         avctx->max_b_frames = MAX_B_FRAMES;
327     }
328     s->max_b_frames = avctx->max_b_frames;
329     s->codec_id     = avctx->codec->id;
330     s->strict_std_compliance = avctx->strict_std_compliance;
331     s->quarter_sample     = (avctx->flags & CODEC_FLAG_QPEL) != 0;
332     s->mpeg_quant         = avctx->mpeg_quant;
333     s->rtp_mode           = !!avctx->rtp_payload_size;
334     s->intra_dc_precision = avctx->intra_dc_precision;
335     s->user_specified_pts = AV_NOPTS_VALUE;
336
337     if (s->gop_size <= 1) {
338         s->intra_only = 1;
339         s->gop_size   = 12;
340     } else {
341         s->intra_only = 0;
342     }
343
344     s->me_method = avctx->me_method;
345
346     /* Fixed QSCALE */
347     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
348
349     s->adaptive_quant = (s->avctx->lumi_masking ||
350                          s->avctx->dark_masking ||
351                          s->avctx->temporal_cplx_masking ||
352                          s->avctx->spatial_cplx_masking  ||
353                          s->avctx->p_masking      ||
354                          s->avctx->border_masking ||
355                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
356                         !s->fixed_qscale;
357
358     s->loop_filter      = !!(s->flags & CODEC_FLAG_LOOP_FILTER);
359
360     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
361         switch(avctx->codec_id) {
362         case AV_CODEC_ID_MPEG1VIDEO:
363         case AV_CODEC_ID_MPEG2VIDEO:
364             avctx->rc_buffer_size = FFMAX(avctx->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
365             break;
366         case AV_CODEC_ID_MPEG4:
367         case AV_CODEC_ID_MSMPEG4V1:
368         case AV_CODEC_ID_MSMPEG4V2:
369         case AV_CODEC_ID_MSMPEG4V3:
370             if       (avctx->rc_max_rate >= 15000000) {
371                 avctx->rc_buffer_size = 320 + (avctx->rc_max_rate - 15000000L) * (760-320) / (38400000 - 15000000);
372             } else if(avctx->rc_max_rate >=  2000000) {
373                 avctx->rc_buffer_size =  80 + (avctx->rc_max_rate -  2000000L) * (320- 80) / (15000000 -  2000000);
374             } else if(avctx->rc_max_rate >=   384000) {
375                 avctx->rc_buffer_size =  40 + (avctx->rc_max_rate -   384000L) * ( 80- 40) / ( 2000000 -   384000);
376             } else
377                 avctx->rc_buffer_size = 40;
378             avctx->rc_buffer_size *= 16384;
379             break;
380         }
381         if (avctx->rc_buffer_size) {
382             av_log(avctx, AV_LOG_INFO, "Automatically choosing VBV buffer size of %d kbyte\n", avctx->rc_buffer_size/8192);
383         }
384     }
385
386     if ((!avctx->rc_max_rate) != (!avctx->rc_buffer_size)) {
387         av_log(avctx, AV_LOG_ERROR, "Either both buffer size and max rate or neither must be specified\n");
388         if (avctx->rc_max_rate && !avctx->rc_buffer_size)
389             return -1;
390     }
391
392     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
393         av_log(avctx, AV_LOG_INFO,
394                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
395     }
396
397     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
398         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
399         return -1;
400     }
401
402     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
403         av_log(avctx, AV_LOG_ERROR, "bitrate above max bitrate\n");
404         return -1;
405     }
406
407     if (avctx->rc_max_rate &&
408         avctx->rc_max_rate == avctx->bit_rate &&
409         avctx->rc_max_rate != avctx->rc_min_rate) {
410         av_log(avctx, AV_LOG_INFO,
411                "impossible bitrate constraints, this will fail\n");
412     }
413
414     if (avctx->rc_buffer_size &&
415         avctx->bit_rate * (int64_t)avctx->time_base.num >
416             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
417         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
418         return -1;
419     }
420
421     if (!s->fixed_qscale &&
422         avctx->bit_rate * av_q2d(avctx->time_base) >
423             avctx->bit_rate_tolerance) {
424         av_log(avctx, AV_LOG_WARNING,
425                "bitrate tolerance %d too small for bitrate %d, overriding\n", avctx->bit_rate_tolerance, avctx->bit_rate);
426         avctx->bit_rate_tolerance = 5 * avctx->bit_rate * av_q2d(avctx->time_base);
427     }
428
429     if (s->avctx->rc_max_rate &&
430         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
431         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
432          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
433         90000LL * (avctx->rc_buffer_size - 1) >
434             s->avctx->rc_max_rate * 0xFFFFLL) {
435         av_log(avctx, AV_LOG_INFO,
436                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
437                "specified vbv buffer is too large for the given bitrate!\n");
438     }
439
440     if ((s->flags & CODEC_FLAG_4MV)  && s->codec_id != AV_CODEC_ID_MPEG4 &&
441         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
442         s->codec_id != AV_CODEC_ID_FLV1) {
443         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
444         return -1;
445     }
446
447     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
448         av_log(avctx, AV_LOG_ERROR,
449                "OBMC is only supported with simple mb decision\n");
450         return -1;
451     }
452
453     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
454         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
455         return -1;
456     }
457
458     if (s->max_b_frames                    &&
459         s->codec_id != AV_CODEC_ID_MPEG4      &&
460         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
461         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
462         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
463         return -1;
464     }
465     if (s->max_b_frames < 0) {
466         av_log(avctx, AV_LOG_ERROR,
467                "max b frames must be 0 or positive for mpegvideo based encoders\n");
468         return -1;
469     }
470
471     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
472          s->codec_id == AV_CODEC_ID_H263  ||
473          s->codec_id == AV_CODEC_ID_H263P) &&
474         (avctx->sample_aspect_ratio.num > 255 ||
475          avctx->sample_aspect_ratio.den > 255)) {
476         av_log(avctx, AV_LOG_WARNING,
477                "Invalid pixel aspect ratio %i/%i, limit is 255/255 reducing\n",
478                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
479         av_reduce(&avctx->sample_aspect_ratio.num, &avctx->sample_aspect_ratio.den,
480                    avctx->sample_aspect_ratio.num,  avctx->sample_aspect_ratio.den, 255);
481     }
482
483     if ((s->codec_id == AV_CODEC_ID_H263  ||
484          s->codec_id == AV_CODEC_ID_H263P) &&
485         (avctx->width  > 2048 ||
486          avctx->height > 1152 )) {
487         av_log(avctx, AV_LOG_ERROR, "H.263 does not support resolutions above 2048x1152\n");
488         return -1;
489     }
490     if ((s->codec_id == AV_CODEC_ID_H263  ||
491          s->codec_id == AV_CODEC_ID_H263P) &&
492         ((avctx->width &3) ||
493          (avctx->height&3) )) {
494         av_log(avctx, AV_LOG_ERROR, "w/h must be a multiple of 4\n");
495         return -1;
496     }
497
498     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO &&
499         (avctx->width  > 4095 ||
500          avctx->height > 4095 )) {
501         av_log(avctx, AV_LOG_ERROR, "MPEG-1 does not support resolutions above 4095x4095\n");
502         return -1;
503     }
504
505     if (s->codec_id == AV_CODEC_ID_MPEG2VIDEO &&
506         (avctx->width  > 16383 ||
507          avctx->height > 16383 )) {
508         av_log(avctx, AV_LOG_ERROR, "MPEG-2 does not support resolutions above 16383x16383\n");
509         return -1;
510     }
511
512     if (s->codec_id == AV_CODEC_ID_RV10 &&
513         (avctx->width &15 ||
514          avctx->height&15 )) {
515         av_log(avctx, AV_LOG_ERROR, "width and height must be a multiple of 16\n");
516         return AVERROR(EINVAL);
517     }
518
519     if (s->codec_id == AV_CODEC_ID_RV20 &&
520         (avctx->width &3 ||
521          avctx->height&3 )) {
522         av_log(avctx, AV_LOG_ERROR, "width and height must be a multiple of 4\n");
523         return AVERROR(EINVAL);
524     }
525
526     if ((s->codec_id == AV_CODEC_ID_WMV1 ||
527          s->codec_id == AV_CODEC_ID_WMV2) &&
528          avctx->width & 1) {
529          av_log(avctx, AV_LOG_ERROR, "width must be multiple of 2\n");
530          return -1;
531     }
532
533     if ((s->flags & (CODEC_FLAG_INTERLACED_DCT | CODEC_FLAG_INTERLACED_ME)) &&
534         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
535         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
536         return -1;
537     }
538
539     // FIXME mpeg2 uses that too
540     if (s->mpeg_quant && (   s->codec_id != AV_CODEC_ID_MPEG4
541                           && s->codec_id != AV_CODEC_ID_MPEG2VIDEO)) {
542         av_log(avctx, AV_LOG_ERROR,
543                "mpeg2 style quantization not supported by codec\n");
544         return -1;
545     }
546
547     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
548         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
549         return -1;
550     }
551
552     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
553         s->avctx->mb_decision != FF_MB_DECISION_RD) {
554         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
555         return -1;
556     }
557
558     if (s->avctx->scenechange_threshold < 1000000000 &&
559         (s->flags & CODEC_FLAG_CLOSED_GOP)) {
560         av_log(avctx, AV_LOG_ERROR,
561                "closed gop with scene change detection are not supported yet, "
562                "set threshold to 1000000000\n");
563         return -1;
564     }
565
566     if (s->flags & CODEC_FLAG_LOW_DELAY) {
567         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
568             av_log(avctx, AV_LOG_ERROR,
569                   "low delay forcing is only available for mpeg2\n");
570             return -1;
571         }
572         if (s->max_b_frames != 0) {
573             av_log(avctx, AV_LOG_ERROR,
574                    "b frames cannot be used with low delay\n");
575             return -1;
576         }
577     }
578
579     if (s->q_scale_type == 1) {
580         if (avctx->qmax > 12) {
581             av_log(avctx, AV_LOG_ERROR,
582                    "non linear quant only supports qmax <= 12 currently\n");
583             return -1;
584         }
585     }
586
587     if (s->avctx->thread_count > 1         &&
588         s->codec_id != AV_CODEC_ID_MPEG4      &&
589         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
590         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
591         s->codec_id != AV_CODEC_ID_MJPEG      &&
592         (s->codec_id != AV_CODEC_ID_H263P)) {
593         av_log(avctx, AV_LOG_ERROR,
594                "multi threaded encoding not supported by codec\n");
595         return -1;
596     }
597
598     if (s->avctx->thread_count < 1) {
599         av_log(avctx, AV_LOG_ERROR,
600                "automatic thread number detection not supported by codec, "
601                "patch welcome\n");
602         return -1;
603     }
604
605     if (s->avctx->slices > 1 || s->avctx->thread_count > 1)
606         s->rtp_mode = 1;
607
608     if (s->avctx->thread_count > 1 && s->codec_id == AV_CODEC_ID_H263P)
609         s->h263_slice_structured = 1;
610
611     if (!avctx->time_base.den || !avctx->time_base.num) {
612         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
613         return -1;
614     }
615
616     i = (INT_MAX / 2 + 128) >> 8;
617     if (avctx->mb_threshold >= i) {
618         av_log(avctx, AV_LOG_ERROR, "mb_threshold too large, max is %d\n",
619                i - 1);
620         return -1;
621     }
622
623     if (avctx->b_frame_strategy && (avctx->flags & CODEC_FLAG_PASS2)) {
624         av_log(avctx, AV_LOG_INFO,
625                "notice: b_frame_strategy only affects the first pass\n");
626         avctx->b_frame_strategy = 0;
627     }
628
629     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
630     if (i > 1) {
631         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
632         avctx->time_base.den /= i;
633         avctx->time_base.num /= i;
634         //return -1;
635     }
636
637     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG || s->codec_id==AV_CODEC_ID_AMV) {
638         // (a + x * 3 / 8) / x
639         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
640         s->inter_quant_bias = 0;
641     } else {
642         s->intra_quant_bias = 0;
643         // (a - x / 4) / x
644         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
645     }
646
647     if (avctx->qmin > avctx->qmax || avctx->qmin <= 0) {
648         av_log(avctx, AV_LOG_ERROR, "qmin and or qmax are invalid, they must be 0 < min <= max\n");
649         return AVERROR(EINVAL);
650     }
651
652     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
653         s->intra_quant_bias = avctx->intra_quant_bias;
654     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
655         s->inter_quant_bias = avctx->inter_quant_bias;
656
657     av_log(avctx, AV_LOG_DEBUG, "intra_quant_bias = %d inter_quant_bias = %d\n",s->intra_quant_bias,s->inter_quant_bias);
658
659     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
660         s->avctx->time_base.den > (1 << 16) - 1) {
661         av_log(avctx, AV_LOG_ERROR,
662                "timebase %d/%d not supported by MPEG 4 standard, "
663                "the maximum admitted value for the timebase denominator "
664                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
665                (1 << 16) - 1);
666         return -1;
667     }
668     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
669
670     switch (avctx->codec->id) {
671     case AV_CODEC_ID_MPEG1VIDEO:
672         s->out_format = FMT_MPEG1;
673         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
674         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
675         break;
676     case AV_CODEC_ID_MPEG2VIDEO:
677         s->out_format = FMT_MPEG1;
678         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
679         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
680         s->rtp_mode   = 1;
681         break;
682     case AV_CODEC_ID_MJPEG:
683     case AV_CODEC_ID_AMV:
684         s->out_format = FMT_MJPEG;
685         s->intra_only = 1; /* force intra only for jpeg */
686         if (!CONFIG_MJPEG_ENCODER ||
687             ff_mjpeg_encode_init(s) < 0)
688             return -1;
689         avctx->delay = 0;
690         s->low_delay = 1;
691         break;
692     case AV_CODEC_ID_H261:
693         if (!CONFIG_H261_ENCODER)
694             return -1;
695         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
696             av_log(avctx, AV_LOG_ERROR,
697                    "The specified picture size of %dx%d is not valid for the "
698                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
699                     s->width, s->height);
700             return -1;
701         }
702         s->out_format = FMT_H261;
703         avctx->delay  = 0;
704         s->low_delay  = 1;
705         break;
706     case AV_CODEC_ID_H263:
707         if (!CONFIG_H263_ENCODER)
708             return -1;
709         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
710                              s->width, s->height) == 8) {
711             av_log(avctx, AV_LOG_ERROR,
712                    "The specified picture size of %dx%d is not valid for "
713                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
714                    "352x288, 704x576, and 1408x1152. "
715                    "Try H.263+.\n", s->width, s->height);
716             return -1;
717         }
718         s->out_format = FMT_H263;
719         avctx->delay  = 0;
720         s->low_delay  = 1;
721         break;
722     case AV_CODEC_ID_H263P:
723         s->out_format = FMT_H263;
724         s->h263_plus  = 1;
725         /* Fx */
726         s->h263_aic        = (avctx->flags & CODEC_FLAG_AC_PRED) ? 1 : 0;
727         s->modified_quant  = s->h263_aic;
728         s->loop_filter     = (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
729         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
730
731         /* /Fx */
732         /* These are just to be sure */
733         avctx->delay = 0;
734         s->low_delay = 1;
735         break;
736     case AV_CODEC_ID_FLV1:
737         s->out_format      = FMT_H263;
738         s->h263_flv        = 2; /* format = 1; 11-bit codes */
739         s->unrestricted_mv = 1;
740         s->rtp_mode  = 0; /* don't allow GOB */
741         avctx->delay = 0;
742         s->low_delay = 1;
743         break;
744     case AV_CODEC_ID_RV10:
745         s->out_format = FMT_H263;
746         avctx->delay  = 0;
747         s->low_delay  = 1;
748         break;
749     case AV_CODEC_ID_RV20:
750         s->out_format      = FMT_H263;
751         avctx->delay       = 0;
752         s->low_delay       = 1;
753         s->modified_quant  = 1;
754         s->h263_aic        = 1;
755         s->h263_plus       = 1;
756         s->loop_filter     = 1;
757         s->unrestricted_mv = 0;
758         break;
759     case AV_CODEC_ID_MPEG4:
760         s->out_format      = FMT_H263;
761         s->h263_pred       = 1;
762         s->unrestricted_mv = 1;
763         s->low_delay       = s->max_b_frames ? 0 : 1;
764         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
765         break;
766     case AV_CODEC_ID_MSMPEG4V2:
767         s->out_format      = FMT_H263;
768         s->h263_pred       = 1;
769         s->unrestricted_mv = 1;
770         s->msmpeg4_version = 2;
771         avctx->delay       = 0;
772         s->low_delay       = 1;
773         break;
774     case AV_CODEC_ID_MSMPEG4V3:
775         s->out_format        = FMT_H263;
776         s->h263_pred         = 1;
777         s->unrestricted_mv   = 1;
778         s->msmpeg4_version   = 3;
779         s->flipflop_rounding = 1;
780         avctx->delay         = 0;
781         s->low_delay         = 1;
782         break;
783     case AV_CODEC_ID_WMV1:
784         s->out_format        = FMT_H263;
785         s->h263_pred         = 1;
786         s->unrestricted_mv   = 1;
787         s->msmpeg4_version   = 4;
788         s->flipflop_rounding = 1;
789         avctx->delay         = 0;
790         s->low_delay         = 1;
791         break;
792     case AV_CODEC_ID_WMV2:
793         s->out_format        = FMT_H263;
794         s->h263_pred         = 1;
795         s->unrestricted_mv   = 1;
796         s->msmpeg4_version   = 5;
797         s->flipflop_rounding = 1;
798         avctx->delay         = 0;
799         s->low_delay         = 1;
800         break;
801     default:
802         return -1;
803     }
804
805     avctx->has_b_frames = !s->low_delay;
806
807     s->encoding = 1;
808
809     s->progressive_frame    =
810     s->progressive_sequence = !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT |
811                                                 CODEC_FLAG_INTERLACED_ME) ||
812                                 s->alternate_scan);
813
814     /* init */
815     if (ff_MPV_common_init(s) < 0)
816         return -1;
817
818     ff_qpeldsp_init(&s->qdsp);
819
820     s->avctx->coded_frame = s->current_picture.f;
821
822     if (s->msmpeg4_version) {
823         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
824                           2 * 2 * (MAX_LEVEL + 1) *
825                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
826     }
827     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
828
829     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
830     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_chroma_intra_matrix, 64 * 32 * sizeof(int), fail);
831     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
832     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
833     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_chroma_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
834     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
835     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
836                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
837     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
838                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
839
840     if (s->avctx->noise_reduction) {
841         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
842                           2 * 64 * sizeof(uint16_t), fail);
843     }
844
845     ff_dct_encode_init(s);
846
847     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
848         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
849
850     s->quant_precision = 5;
851
852     ff_set_cmp(&s->dsp, s->dsp.ildct_cmp, s->avctx->ildct_cmp);
853     ff_set_cmp(&s->dsp, s->dsp.frame_skip_cmp, s->avctx->frame_skip_cmp);
854
855     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
856         ff_h261_encode_init(s);
857     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
858         ff_h263_encode_init(s);
859     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
860         ff_msmpeg4_encode_init(s);
861     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
862         && s->out_format == FMT_MPEG1)
863         ff_mpeg1_encode_init(s);
864
865     /* init q matrix */
866     for (i = 0; i < 64; i++) {
867         int j = s->dsp.idct_permutation[i];
868         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
869             s->mpeg_quant) {
870             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
871             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
872         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
873             s->intra_matrix[j] =
874             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
875         } else {
876             /* mpeg1/2 */
877             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
878             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
879         }
880         if (s->avctx->intra_matrix)
881             s->intra_matrix[j] = s->avctx->intra_matrix[i];
882         if (s->avctx->inter_matrix)
883             s->inter_matrix[j] = s->avctx->inter_matrix[i];
884     }
885
886     /* precompute matrix */
887     /* for mjpeg, we do include qscale in the matrix */
888     if (s->out_format != FMT_MJPEG) {
889         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
890                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
891                           31, 1);
892         ff_convert_matrix(&s->dsp, s->q_inter_matrix, s->q_inter_matrix16,
893                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
894                           31, 0);
895     }
896
897     if (ff_rate_control_init(s) < 0)
898         return -1;
899
900 #if FF_API_ERROR_RATE
901     FF_DISABLE_DEPRECATION_WARNINGS
902     if (avctx->error_rate)
903         s->error_rate = avctx->error_rate;
904     FF_ENABLE_DEPRECATION_WARNINGS;
905 #endif
906
907 #if FF_API_NORMALIZE_AQP
908     FF_DISABLE_DEPRECATION_WARNINGS
909     if (avctx->flags & CODEC_FLAG_NORMALIZE_AQP)
910         s->mpv_flags |= FF_MPV_FLAG_NAQ;
911     FF_ENABLE_DEPRECATION_WARNINGS;
912 #endif
913
914 #if FF_API_MV0
915     FF_DISABLE_DEPRECATION_WARNINGS
916     if (avctx->flags & CODEC_FLAG_MV0)
917         s->mpv_flags |= FF_MPV_FLAG_MV0;
918     FF_ENABLE_DEPRECATION_WARNINGS
919 #endif
920
921     if (avctx->b_frame_strategy == 2) {
922         for (i = 0; i < s->max_b_frames + 2; i++) {
923             s->tmp_frames[i] = av_frame_alloc();
924             if (!s->tmp_frames[i])
925                 return AVERROR(ENOMEM);
926
927             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
928             s->tmp_frames[i]->width  = s->width  >> avctx->brd_scale;
929             s->tmp_frames[i]->height = s->height >> avctx->brd_scale;
930
931             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
932             if (ret < 0)
933                 return ret;
934         }
935     }
936
937     return 0;
938 fail:
939     ff_MPV_encode_end(avctx);
940     return AVERROR_UNKNOWN;
941 }
942
943 av_cold int ff_MPV_encode_end(AVCodecContext *avctx)
944 {
945     MpegEncContext *s = avctx->priv_data;
946     int i;
947
948     ff_rate_control_uninit(s);
949
950     ff_MPV_common_end(s);
951     if (CONFIG_MJPEG_ENCODER &&
952         s->out_format == FMT_MJPEG)
953         ff_mjpeg_encode_close(s);
954
955     av_freep(&avctx->extradata);
956
957     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
958         av_frame_free(&s->tmp_frames[i]);
959
960     ff_free_picture_tables(&s->new_picture);
961     ff_mpeg_unref_picture(s, &s->new_picture);
962
963     av_freep(&s->avctx->stats_out);
964     av_freep(&s->ac_stats);
965
966     if(s->q_chroma_intra_matrix   != s->q_intra_matrix  ) av_freep(&s->q_chroma_intra_matrix);
967     if(s->q_chroma_intra_matrix16 != s->q_intra_matrix16) av_freep(&s->q_chroma_intra_matrix16);
968     s->q_chroma_intra_matrix=   NULL;
969     s->q_chroma_intra_matrix16= NULL;
970     av_freep(&s->q_intra_matrix);
971     av_freep(&s->q_inter_matrix);
972     av_freep(&s->q_intra_matrix16);
973     av_freep(&s->q_inter_matrix16);
974     av_freep(&s->input_picture);
975     av_freep(&s->reordered_input_picture);
976     av_freep(&s->dct_offset);
977
978     return 0;
979 }
980
981 static int get_sae(uint8_t *src, int ref, int stride)
982 {
983     int x,y;
984     int acc = 0;
985
986     for (y = 0; y < 16; y++) {
987         for (x = 0; x < 16; x++) {
988             acc += FFABS(src[x + y * stride] - ref);
989         }
990     }
991
992     return acc;
993 }
994
995 static int get_intra_count(MpegEncContext *s, uint8_t *src,
996                            uint8_t *ref, int stride)
997 {
998     int x, y, w, h;
999     int acc = 0;
1000
1001     w = s->width  & ~15;
1002     h = s->height & ~15;
1003
1004     for (y = 0; y < h; y += 16) {
1005         for (x = 0; x < w; x += 16) {
1006             int offset = x + y * stride;
1007             int sad  = s->dsp.sad[0](NULL, src + offset, ref + offset, stride,
1008                                      16);
1009             int mean = (s->dsp.pix_sum(src + offset, stride) + 128) >> 8;
1010             int sae  = get_sae(src + offset, mean, stride);
1011
1012             acc += sae + 500 < sad;
1013         }
1014     }
1015     return acc;
1016 }
1017
1018
1019 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
1020 {
1021     Picture *pic = NULL;
1022     int64_t pts;
1023     int i, display_picture_number = 0, ret;
1024     const int encoding_delay = s->max_b_frames ? s->max_b_frames :
1025                                                  (s->low_delay ? 0 : 1);
1026     int direct = 1;
1027
1028     if (pic_arg) {
1029         pts = pic_arg->pts;
1030         display_picture_number = s->input_picture_number++;
1031
1032         if (pts != AV_NOPTS_VALUE) {
1033             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1034                 int64_t last = s->user_specified_pts;
1035
1036                 if (pts <= last) {
1037                     av_log(s->avctx, AV_LOG_ERROR,
1038                            "Invalid pts (%"PRId64") <= last (%"PRId64")\n",
1039                            pts, last);
1040                     return AVERROR(EINVAL);
1041                 }
1042
1043                 if (!s->low_delay && display_picture_number == 1)
1044                     s->dts_delta = pts - last;
1045             }
1046             s->user_specified_pts = pts;
1047         } else {
1048             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1049                 s->user_specified_pts =
1050                 pts = s->user_specified_pts + 1;
1051                 av_log(s->avctx, AV_LOG_INFO,
1052                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
1053                        pts);
1054             } else {
1055                 pts = display_picture_number;
1056             }
1057         }
1058     }
1059
1060     if (pic_arg) {
1061         if (!pic_arg->buf[0])
1062             direct = 0;
1063         if (pic_arg->linesize[0] != s->linesize)
1064             direct = 0;
1065         if (pic_arg->linesize[1] != s->uvlinesize)
1066             direct = 0;
1067         if (pic_arg->linesize[2] != s->uvlinesize)
1068             direct = 0;
1069         if ((s->width & 15) || (s->height & 15))
1070             direct = 0;
1071         if (((intptr_t)(pic_arg->data[0])) & (STRIDE_ALIGN-1))
1072             direct = 0;
1073         if (s->linesize & (STRIDE_ALIGN-1))
1074             direct = 0;
1075
1076         av_dlog(s->avctx, "%d %d %"PTRDIFF_SPECIFIER" %"PTRDIFF_SPECIFIER"\n", pic_arg->linesize[0],
1077                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
1078
1079         if (direct) {
1080             i = ff_find_unused_picture(s, 1);
1081             if (i < 0)
1082                 return i;
1083
1084             pic = &s->picture[i];
1085             pic->reference = 3;
1086
1087             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
1088                 return ret;
1089             if (ff_alloc_picture(s, pic, 1) < 0) {
1090                 return -1;
1091             }
1092         } else {
1093             i = ff_find_unused_picture(s, 0);
1094             if (i < 0)
1095                 return i;
1096
1097             pic = &s->picture[i];
1098             pic->reference = 3;
1099
1100             if (ff_alloc_picture(s, pic, 0) < 0) {
1101                 return -1;
1102             }
1103
1104             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1105                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1106                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1107                 // empty
1108             } else {
1109                 int h_chroma_shift, v_chroma_shift;
1110                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
1111                                                  &h_chroma_shift,
1112                                                  &v_chroma_shift);
1113
1114                 for (i = 0; i < 3; i++) {
1115                     int src_stride = pic_arg->linesize[i];
1116                     int dst_stride = i ? s->uvlinesize : s->linesize;
1117                     int h_shift = i ? h_chroma_shift : 0;
1118                     int v_shift = i ? v_chroma_shift : 0;
1119                     int w = s->width  >> h_shift;
1120                     int h = s->height >> v_shift;
1121                     uint8_t *src = pic_arg->data[i];
1122                     uint8_t *dst = pic->f->data[i];
1123
1124                     if (!s->avctx->rc_buffer_size)
1125                         dst += INPLACE_OFFSET;
1126
1127                     if (src_stride == dst_stride)
1128                         memcpy(dst, src, src_stride * h);
1129                     else {
1130                         int h2 = h;
1131                         uint8_t *dst2 = dst;
1132                         while (h2--) {
1133                             memcpy(dst2, src, w);
1134                             dst2 += dst_stride;
1135                             src += src_stride;
1136                         }
1137                     }
1138                     if ((s->width & 15) || (s->height & 15)) {
1139                         s->dsp.draw_edges(dst, dst_stride,
1140                                           w, h,
1141                                           16>>h_shift,
1142                                           16>>v_shift,
1143                                           EDGE_BOTTOM);
1144                     }
1145                 }
1146             }
1147         }
1148         ret = av_frame_copy_props(pic->f, pic_arg);
1149         if (ret < 0)
1150             return ret;
1151
1152         pic->f->display_picture_number = display_picture_number;
1153         pic->f->pts = pts; // we set this here to avoid modifiying pic_arg
1154     }
1155
1156     /* shift buffer entries */
1157     for (i = 1; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1158         s->input_picture[i - 1] = s->input_picture[i];
1159
1160     s->input_picture[encoding_delay] = (Picture*) pic;
1161
1162     return 0;
1163 }
1164
1165 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1166 {
1167     int x, y, plane;
1168     int score = 0;
1169     int64_t score64 = 0;
1170
1171     for (plane = 0; plane < 3; plane++) {
1172         const int stride = p->f->linesize[plane];
1173         const int bw = plane ? 1 : 2;
1174         for (y = 0; y < s->mb_height * bw; y++) {
1175             for (x = 0; x < s->mb_width * bw; x++) {
1176                 int off = p->shared ? 0 : 16;
1177                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1178                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1179                 int v   = s->dsp.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1180
1181                 switch (FFABS(s->avctx->frame_skip_exp)) {
1182                 case 0: score    =  FFMAX(score, v);          break;
1183                 case 1: score   += FFABS(v);                  break;
1184                 case 2: score64 += v * (int64_t)v;                       break;
1185                 case 3: score64 += FFABS(v * (int64_t)v * v);            break;
1186                 case 4: score64 += (v * (int64_t)v) * (v * (int64_t)v);  break;
1187                 }
1188             }
1189         }
1190     }
1191     emms_c();
1192
1193     if (score)
1194         score64 = score;
1195     if (s->avctx->frame_skip_exp < 0)
1196         score64 = pow(score64 / (double)(s->mb_width * s->mb_height),
1197                       -1.0/s->avctx->frame_skip_exp);
1198
1199     if (score64 < s->avctx->frame_skip_threshold)
1200         return 1;
1201     if (score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda) >> 8))
1202         return 1;
1203     return 0;
1204 }
1205
1206 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1207 {
1208     AVPacket pkt = { 0 };
1209     int ret, got_output;
1210
1211     av_init_packet(&pkt);
1212     ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
1213     if (ret < 0)
1214         return ret;
1215
1216     ret = pkt.size;
1217     av_free_packet(&pkt);
1218     return ret;
1219 }
1220
1221 static int estimate_best_b_count(MpegEncContext *s)
1222 {
1223     AVCodec *codec    = avcodec_find_encoder(s->avctx->codec_id);
1224     AVCodecContext *c = avcodec_alloc_context3(NULL);
1225     const int scale = s->avctx->brd_scale;
1226     int i, j, out_size, p_lambda, b_lambda, lambda2;
1227     int64_t best_rd  = INT64_MAX;
1228     int best_b_count = -1;
1229
1230     av_assert0(scale >= 0 && scale <= 3);
1231
1232     //emms_c();
1233     //s->next_picture_ptr->quality;
1234     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1235     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1236     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1237     if (!b_lambda) // FIXME we should do this somewhere else
1238         b_lambda = p_lambda;
1239     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1240                FF_LAMBDA_SHIFT;
1241
1242     c->width        = s->width  >> scale;
1243     c->height       = s->height >> scale;
1244     c->flags        = CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR;
1245     c->flags       |= s->avctx->flags & CODEC_FLAG_QPEL;
1246     c->mb_decision  = s->avctx->mb_decision;
1247     c->me_cmp       = s->avctx->me_cmp;
1248     c->mb_cmp       = s->avctx->mb_cmp;
1249     c->me_sub_cmp   = s->avctx->me_sub_cmp;
1250     c->pix_fmt      = AV_PIX_FMT_YUV420P;
1251     c->time_base    = s->avctx->time_base;
1252     c->max_b_frames = s->max_b_frames;
1253
1254     if (avcodec_open2(c, codec, NULL) < 0)
1255         return -1;
1256
1257     for (i = 0; i < s->max_b_frames + 2; i++) {
1258         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1259                                                 s->next_picture_ptr;
1260
1261         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1262             pre_input = *pre_input_ptr;
1263
1264             if (!pre_input.shared && i) {
1265                 pre_input.f->data[0] += INPLACE_OFFSET;
1266                 pre_input.f->data[1] += INPLACE_OFFSET;
1267                 pre_input.f->data[2] += INPLACE_OFFSET;
1268             }
1269
1270             s->dsp.shrink[scale](s->tmp_frames[i]->data[0], s->tmp_frames[i]->linesize[0],
1271                                  pre_input.f->data[0], pre_input.f->linesize[0],
1272                                  c->width,      c->height);
1273             s->dsp.shrink[scale](s->tmp_frames[i]->data[1], s->tmp_frames[i]->linesize[1],
1274                                  pre_input.f->data[1], pre_input.f->linesize[1],
1275                                  c->width >> 1, c->height >> 1);
1276             s->dsp.shrink[scale](s->tmp_frames[i]->data[2], s->tmp_frames[i]->linesize[2],
1277                                  pre_input.f->data[2], pre_input.f->linesize[2],
1278                                  c->width >> 1, c->height >> 1);
1279         }
1280     }
1281
1282     for (j = 0; j < s->max_b_frames + 1; j++) {
1283         int64_t rd = 0;
1284
1285         if (!s->input_picture[j])
1286             break;
1287
1288         c->error[0] = c->error[1] = c->error[2] = 0;
1289
1290         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1291         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1292
1293         out_size = encode_frame(c, s->tmp_frames[0]);
1294
1295         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1296
1297         for (i = 0; i < s->max_b_frames + 1; i++) {
1298             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1299
1300             s->tmp_frames[i + 1]->pict_type = is_p ?
1301                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1302             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1303
1304             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1305
1306             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1307         }
1308
1309         /* get the delayed frames */
1310         while (out_size) {
1311             out_size = encode_frame(c, NULL);
1312             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1313         }
1314
1315         rd += c->error[0] + c->error[1] + c->error[2];
1316
1317         if (rd < best_rd) {
1318             best_rd = rd;
1319             best_b_count = j;
1320         }
1321     }
1322
1323     avcodec_close(c);
1324     av_freep(&c);
1325
1326     return best_b_count;
1327 }
1328
1329 static int select_input_picture(MpegEncContext *s)
1330 {
1331     int i, ret;
1332
1333     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1334         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1335     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1336
1337     /* set next picture type & ordering */
1338     if (s->reordered_input_picture[0] == NULL && s->input_picture[0]) {
1339         if (s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor) {
1340             if (s->picture_in_gop_number < s->gop_size &&
1341                 s->next_picture_ptr &&
1342                 skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1343                 // FIXME check that te gop check above is +-1 correct
1344                 av_frame_unref(s->input_picture[0]->f);
1345
1346                 ff_vbv_update(s, 0);
1347
1348                 goto no_output_pic;
1349             }
1350         }
1351
1352         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1353             s->next_picture_ptr == NULL || s->intra_only) {
1354             s->reordered_input_picture[0] = s->input_picture[0];
1355             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1356             s->reordered_input_picture[0]->f->coded_picture_number =
1357                 s->coded_picture_number++;
1358         } else {
1359             int b_frames;
1360
1361             if (s->flags & CODEC_FLAG_PASS2) {
1362                 for (i = 0; i < s->max_b_frames + 1; i++) {
1363                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1364
1365                     if (pict_num >= s->rc_context.num_entries)
1366                         break;
1367                     if (!s->input_picture[i]) {
1368                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1369                         break;
1370                     }
1371
1372                     s->input_picture[i]->f->pict_type =
1373                         s->rc_context.entry[pict_num].new_pict_type;
1374                 }
1375             }
1376
1377             if (s->avctx->b_frame_strategy == 0) {
1378                 b_frames = s->max_b_frames;
1379                 while (b_frames && !s->input_picture[b_frames])
1380                     b_frames--;
1381             } else if (s->avctx->b_frame_strategy == 1) {
1382                 for (i = 1; i < s->max_b_frames + 1; i++) {
1383                     if (s->input_picture[i] &&
1384                         s->input_picture[i]->b_frame_score == 0) {
1385                         s->input_picture[i]->b_frame_score =
1386                             get_intra_count(s,
1387                                             s->input_picture[i    ]->f->data[0],
1388                                             s->input_picture[i - 1]->f->data[0],
1389                                             s->linesize) + 1;
1390                     }
1391                 }
1392                 for (i = 0; i < s->max_b_frames + 1; i++) {
1393                     if (s->input_picture[i] == NULL ||
1394                         s->input_picture[i]->b_frame_score - 1 >
1395                             s->mb_num / s->avctx->b_sensitivity)
1396                         break;
1397                 }
1398
1399                 b_frames = FFMAX(0, i - 1);
1400
1401                 /* reset scores */
1402                 for (i = 0; i < b_frames + 1; i++) {
1403                     s->input_picture[i]->b_frame_score = 0;
1404                 }
1405             } else if (s->avctx->b_frame_strategy == 2) {
1406                 b_frames = estimate_best_b_count(s);
1407             } else {
1408                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1409                 b_frames = 0;
1410             }
1411
1412             emms_c();
1413
1414             for (i = b_frames - 1; i >= 0; i--) {
1415                 int type = s->input_picture[i]->f->pict_type;
1416                 if (type && type != AV_PICTURE_TYPE_B)
1417                     b_frames = i;
1418             }
1419             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1420                 b_frames == s->max_b_frames) {
1421                 av_log(s->avctx, AV_LOG_ERROR,
1422                        "warning, too many b frames in a row\n");
1423             }
1424
1425             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1426                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1427                     s->gop_size > s->picture_in_gop_number) {
1428                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1429                 } else {
1430                     if (s->flags & CODEC_FLAG_CLOSED_GOP)
1431                         b_frames = 0;
1432                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1433                 }
1434             }
1435
1436             if ((s->flags & CODEC_FLAG_CLOSED_GOP) && b_frames &&
1437                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1438                 b_frames--;
1439
1440             s->reordered_input_picture[0] = s->input_picture[b_frames];
1441             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1442                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1443             s->reordered_input_picture[0]->f->coded_picture_number =
1444                 s->coded_picture_number++;
1445             for (i = 0; i < b_frames; i++) {
1446                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1447                 s->reordered_input_picture[i + 1]->f->pict_type =
1448                     AV_PICTURE_TYPE_B;
1449                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1450                     s->coded_picture_number++;
1451             }
1452         }
1453     }
1454 no_output_pic:
1455     if (s->reordered_input_picture[0]) {
1456         s->reordered_input_picture[0]->reference =
1457            s->reordered_input_picture[0]->f->pict_type !=
1458                AV_PICTURE_TYPE_B ? 3 : 0;
1459
1460         ff_mpeg_unref_picture(s, &s->new_picture);
1461         if ((ret = ff_mpeg_ref_picture(s, &s->new_picture, s->reordered_input_picture[0])))
1462             return ret;
1463
1464         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1465             // input is a shared pix, so we can't modifiy it -> alloc a new
1466             // one & ensure that the shared one is reuseable
1467
1468             Picture *pic;
1469             int i = ff_find_unused_picture(s, 0);
1470             if (i < 0)
1471                 return i;
1472             pic = &s->picture[i];
1473
1474             pic->reference = s->reordered_input_picture[0]->reference;
1475             if (ff_alloc_picture(s, pic, 0) < 0) {
1476                 return -1;
1477             }
1478
1479             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1480             if (ret < 0)
1481                 return ret;
1482
1483             /* mark us unused / free shared pic */
1484             av_frame_unref(s->reordered_input_picture[0]->f);
1485             s->reordered_input_picture[0]->shared = 0;
1486
1487             s->current_picture_ptr = pic;
1488         } else {
1489             // input is not a shared pix -> reuse buffer for current_pix
1490             s->current_picture_ptr = s->reordered_input_picture[0];
1491             for (i = 0; i < 4; i++) {
1492                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1493             }
1494         }
1495         ff_mpeg_unref_picture(s, &s->current_picture);
1496         if ((ret = ff_mpeg_ref_picture(s, &s->current_picture,
1497                                        s->current_picture_ptr)) < 0)
1498             return ret;
1499
1500         s->picture_number = s->new_picture.f->display_picture_number;
1501     } else {
1502         ff_mpeg_unref_picture(s, &s->new_picture);
1503     }
1504     return 0;
1505 }
1506
1507 static void frame_end(MpegEncContext *s)
1508 {
1509     if (s->unrestricted_mv &&
1510         s->current_picture.reference &&
1511         !s->intra_only) {
1512         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1513         int hshift = desc->log2_chroma_w;
1514         int vshift = desc->log2_chroma_h;
1515         s->dsp.draw_edges(s->current_picture.f->data[0], s->current_picture.f->linesize[0],
1516                           s->h_edge_pos, s->v_edge_pos,
1517                           EDGE_WIDTH, EDGE_WIDTH,
1518                           EDGE_TOP | EDGE_BOTTOM);
1519         s->dsp.draw_edges(s->current_picture.f->data[1], s->current_picture.f->linesize[1],
1520                           s->h_edge_pos >> hshift, s->v_edge_pos >> vshift,
1521                           EDGE_WIDTH >> hshift, EDGE_WIDTH >> vshift,
1522                           EDGE_TOP | EDGE_BOTTOM);
1523         s->dsp.draw_edges(s->current_picture.f->data[2], s->current_picture.f->linesize[2],
1524                           s->h_edge_pos >> hshift, s->v_edge_pos >> vshift,
1525                           EDGE_WIDTH >> hshift, EDGE_WIDTH >> vshift,
1526                           EDGE_TOP | EDGE_BOTTOM);
1527     }
1528
1529     emms_c();
1530
1531     s->last_pict_type                 = s->pict_type;
1532     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1533     if (s->pict_type!= AV_PICTURE_TYPE_B)
1534         s->last_non_b_pict_type = s->pict_type;
1535
1536     s->avctx->coded_frame = s->current_picture_ptr->f;
1537
1538 }
1539
1540 static void update_noise_reduction(MpegEncContext *s)
1541 {
1542     int intra, i;
1543
1544     for (intra = 0; intra < 2; intra++) {
1545         if (s->dct_count[intra] > (1 << 16)) {
1546             for (i = 0; i < 64; i++) {
1547                 s->dct_error_sum[intra][i] >>= 1;
1548             }
1549             s->dct_count[intra] >>= 1;
1550         }
1551
1552         for (i = 0; i < 64; i++) {
1553             s->dct_offset[intra][i] = (s->avctx->noise_reduction *
1554                                        s->dct_count[intra] +
1555                                        s->dct_error_sum[intra][i] / 2) /
1556                                       (s->dct_error_sum[intra][i] + 1);
1557         }
1558     }
1559 }
1560
1561 static int frame_start(MpegEncContext *s)
1562 {
1563     int ret;
1564
1565     /* mark & release old frames */
1566     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1567         s->last_picture_ptr != s->next_picture_ptr &&
1568         s->last_picture_ptr->f->buf[0]) {
1569         ff_mpeg_unref_picture(s, s->last_picture_ptr);
1570     }
1571
1572     s->current_picture_ptr->f->pict_type = s->pict_type;
1573     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1574
1575     ff_mpeg_unref_picture(s, &s->current_picture);
1576     if ((ret = ff_mpeg_ref_picture(s, &s->current_picture,
1577                                    s->current_picture_ptr)) < 0)
1578         return ret;
1579
1580     if (s->pict_type != AV_PICTURE_TYPE_B) {
1581         s->last_picture_ptr = s->next_picture_ptr;
1582         if (!s->droppable)
1583             s->next_picture_ptr = s->current_picture_ptr;
1584     }
1585
1586     if (s->last_picture_ptr) {
1587         ff_mpeg_unref_picture(s, &s->last_picture);
1588         if (s->last_picture_ptr->f->buf[0] &&
1589             (ret = ff_mpeg_ref_picture(s, &s->last_picture,
1590                                        s->last_picture_ptr)) < 0)
1591             return ret;
1592     }
1593     if (s->next_picture_ptr) {
1594         ff_mpeg_unref_picture(s, &s->next_picture);
1595         if (s->next_picture_ptr->f->buf[0] &&
1596             (ret = ff_mpeg_ref_picture(s, &s->next_picture,
1597                                        s->next_picture_ptr)) < 0)
1598             return ret;
1599     }
1600
1601     if (s->picture_structure!= PICT_FRAME) {
1602         int i;
1603         for (i = 0; i < 4; i++) {
1604             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1605                 s->current_picture.f->data[i] +=
1606                     s->current_picture.f->linesize[i];
1607             }
1608             s->current_picture.f->linesize[i] *= 2;
1609             s->last_picture.f->linesize[i]    *= 2;
1610             s->next_picture.f->linesize[i]    *= 2;
1611         }
1612     }
1613
1614     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1615         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1616         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1617     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1618         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1619         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1620     } else {
1621         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1622         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1623     }
1624
1625     if (s->dct_error_sum) {
1626         av_assert2(s->avctx->noise_reduction && s->encoding);
1627         update_noise_reduction(s);
1628     }
1629
1630     return 0;
1631 }
1632
1633 int ff_MPV_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1634                           const AVFrame *pic_arg, int *got_packet)
1635 {
1636     MpegEncContext *s = avctx->priv_data;
1637     int i, stuffing_count, ret;
1638     int context_count = s->slice_context_count;
1639
1640     s->picture_in_gop_number++;
1641
1642     if (load_input_picture(s, pic_arg) < 0)
1643         return -1;
1644
1645     if (select_input_picture(s) < 0) {
1646         return -1;
1647     }
1648
1649     /* output? */
1650     if (s->new_picture.f->data[0]) {
1651         if ((ret = ff_alloc_packet2(avctx, pkt, s->mb_width*s->mb_height*(MAX_MB_BYTES+100)+10000)) < 0)
1652             return ret;
1653         if (s->mb_info) {
1654             s->mb_info_ptr = av_packet_new_side_data(pkt,
1655                                  AV_PKT_DATA_H263_MB_INFO,
1656                                  s->mb_width*s->mb_height*12);
1657             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1658         }
1659
1660         for (i = 0; i < context_count; i++) {
1661             int start_y = s->thread_context[i]->start_mb_y;
1662             int   end_y = s->thread_context[i]->  end_mb_y;
1663             int h       = s->mb_height;
1664             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1665             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1666
1667             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1668         }
1669
1670         s->pict_type = s->new_picture.f->pict_type;
1671         //emms_c();
1672         ret = frame_start(s);
1673         if (ret < 0)
1674             return ret;
1675 vbv_retry:
1676         if (encode_picture(s, s->picture_number) < 0)
1677             return -1;
1678
1679         avctx->header_bits = s->header_bits;
1680         avctx->mv_bits     = s->mv_bits;
1681         avctx->misc_bits   = s->misc_bits;
1682         avctx->i_tex_bits  = s->i_tex_bits;
1683         avctx->p_tex_bits  = s->p_tex_bits;
1684         avctx->i_count     = s->i_count;
1685         // FIXME f/b_count in avctx
1686         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1687         avctx->skip_count  = s->skip_count;
1688
1689         frame_end(s);
1690
1691         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1692             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1693
1694         if (avctx->rc_buffer_size) {
1695             RateControlContext *rcc = &s->rc_context;
1696             int max_size = rcc->buffer_index * avctx->rc_max_available_vbv_use;
1697
1698             if (put_bits_count(&s->pb) > max_size &&
1699                 s->lambda < s->avctx->lmax) {
1700                 s->next_lambda = FFMAX(s->lambda + 1, s->lambda *
1701                                        (s->qscale + 1) / s->qscale);
1702                 if (s->adaptive_quant) {
1703                     int i;
1704                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1705                         s->lambda_table[i] =
1706                             FFMAX(s->lambda_table[i] + 1,
1707                                   s->lambda_table[i] * (s->qscale + 1) /
1708                                   s->qscale);
1709                 }
1710                 s->mb_skipped = 0;        // done in frame_start()
1711                 // done in encode_picture() so we must undo it
1712                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1713                     if (s->flipflop_rounding          ||
1714                         s->codec_id == AV_CODEC_ID_H263P ||
1715                         s->codec_id == AV_CODEC_ID_MPEG4)
1716                         s->no_rounding ^= 1;
1717                 }
1718                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1719                     s->time_base       = s->last_time_base;
1720                     s->last_non_b_time = s->time - s->pp_time;
1721                 }
1722                 for (i = 0; i < context_count; i++) {
1723                     PutBitContext *pb = &s->thread_context[i]->pb;
1724                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1725                 }
1726                 goto vbv_retry;
1727             }
1728
1729             av_assert0(s->avctx->rc_max_rate);
1730         }
1731
1732         if (s->flags & CODEC_FLAG_PASS1)
1733             ff_write_pass1_stats(s);
1734
1735         for (i = 0; i < 4; i++) {
1736             s->current_picture_ptr->f->error[i] = s->current_picture.f->error[i];
1737             avctx->error[i] += s->current_picture_ptr->f->error[i];
1738         }
1739
1740         if (s->flags & CODEC_FLAG_PASS1)
1741             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits +
1742                    avctx->i_tex_bits + avctx->p_tex_bits ==
1743                        put_bits_count(&s->pb));
1744         flush_put_bits(&s->pb);
1745         s->frame_bits  = put_bits_count(&s->pb);
1746
1747         stuffing_count = ff_vbv_update(s, s->frame_bits);
1748         s->stuffing_bits = 8*stuffing_count;
1749         if (stuffing_count) {
1750             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1751                     stuffing_count + 50) {
1752                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1753                 return -1;
1754             }
1755
1756             switch (s->codec_id) {
1757             case AV_CODEC_ID_MPEG1VIDEO:
1758             case AV_CODEC_ID_MPEG2VIDEO:
1759                 while (stuffing_count--) {
1760                     put_bits(&s->pb, 8, 0);
1761                 }
1762             break;
1763             case AV_CODEC_ID_MPEG4:
1764                 put_bits(&s->pb, 16, 0);
1765                 put_bits(&s->pb, 16, 0x1C3);
1766                 stuffing_count -= 4;
1767                 while (stuffing_count--) {
1768                     put_bits(&s->pb, 8, 0xFF);
1769                 }
1770             break;
1771             default:
1772                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1773             }
1774             flush_put_bits(&s->pb);
1775             s->frame_bits  = put_bits_count(&s->pb);
1776         }
1777
1778         /* update mpeg1/2 vbv_delay for CBR */
1779         if (s->avctx->rc_max_rate                          &&
1780             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1781             s->out_format == FMT_MPEG1                     &&
1782             90000LL * (avctx->rc_buffer_size - 1) <=
1783                 s->avctx->rc_max_rate * 0xFFFFLL) {
1784             int vbv_delay, min_delay;
1785             double inbits  = s->avctx->rc_max_rate *
1786                              av_q2d(s->avctx->time_base);
1787             int    minbits = s->frame_bits - 8 *
1788                              (s->vbv_delay_ptr - s->pb.buf - 1);
1789             double bits    = s->rc_context.buffer_index + minbits - inbits;
1790
1791             if (bits < 0)
1792                 av_log(s->avctx, AV_LOG_ERROR,
1793                        "Internal error, negative bits\n");
1794
1795             assert(s->repeat_first_field == 0);
1796
1797             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
1798             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
1799                         s->avctx->rc_max_rate;
1800
1801             vbv_delay = FFMAX(vbv_delay, min_delay);
1802
1803             av_assert0(vbv_delay < 0xFFFF);
1804
1805             s->vbv_delay_ptr[0] &= 0xF8;
1806             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
1807             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
1808             s->vbv_delay_ptr[2] &= 0x07;
1809             s->vbv_delay_ptr[2] |= vbv_delay << 3;
1810             avctx->vbv_delay     = vbv_delay * 300;
1811         }
1812         s->total_bits     += s->frame_bits;
1813         avctx->frame_bits  = s->frame_bits;
1814
1815         pkt->pts = s->current_picture.f->pts;
1816         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
1817             if (!s->current_picture.f->coded_picture_number)
1818                 pkt->dts = pkt->pts - s->dts_delta;
1819             else
1820                 pkt->dts = s->reordered_pts;
1821             s->reordered_pts = pkt->pts;
1822         } else
1823             pkt->dts = pkt->pts;
1824         if (s->current_picture.f->key_frame)
1825             pkt->flags |= AV_PKT_FLAG_KEY;
1826         if (s->mb_info)
1827             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
1828     } else {
1829         s->frame_bits = 0;
1830     }
1831
1832     /* release non-reference frames */
1833     for (i = 0; i < MAX_PICTURE_COUNT; i++) {
1834         if (!s->picture[i].reference)
1835             ff_mpeg_unref_picture(s, &s->picture[i]);
1836     }
1837
1838     av_assert1((s->frame_bits & 7) == 0);
1839
1840     pkt->size = s->frame_bits / 8;
1841     *got_packet = !!pkt->size;
1842     return 0;
1843 }
1844
1845 static inline void dct_single_coeff_elimination(MpegEncContext *s,
1846                                                 int n, int threshold)
1847 {
1848     static const char tab[64] = {
1849         3, 2, 2, 1, 1, 1, 1, 1,
1850         1, 1, 1, 1, 1, 1, 1, 1,
1851         1, 1, 1, 1, 1, 1, 1, 1,
1852         0, 0, 0, 0, 0, 0, 0, 0,
1853         0, 0, 0, 0, 0, 0, 0, 0,
1854         0, 0, 0, 0, 0, 0, 0, 0,
1855         0, 0, 0, 0, 0, 0, 0, 0,
1856         0, 0, 0, 0, 0, 0, 0, 0
1857     };
1858     int score = 0;
1859     int run = 0;
1860     int i;
1861     int16_t *block = s->block[n];
1862     const int last_index = s->block_last_index[n];
1863     int skip_dc;
1864
1865     if (threshold < 0) {
1866         skip_dc = 0;
1867         threshold = -threshold;
1868     } else
1869         skip_dc = 1;
1870
1871     /* Are all we could set to zero already zero? */
1872     if (last_index <= skip_dc - 1)
1873         return;
1874
1875     for (i = 0; i <= last_index; i++) {
1876         const int j = s->intra_scantable.permutated[i];
1877         const int level = FFABS(block[j]);
1878         if (level == 1) {
1879             if (skip_dc && i == 0)
1880                 continue;
1881             score += tab[run];
1882             run = 0;
1883         } else if (level > 1) {
1884             return;
1885         } else {
1886             run++;
1887         }
1888     }
1889     if (score >= threshold)
1890         return;
1891     for (i = skip_dc; i <= last_index; i++) {
1892         const int j = s->intra_scantable.permutated[i];
1893         block[j] = 0;
1894     }
1895     if (block[0])
1896         s->block_last_index[n] = 0;
1897     else
1898         s->block_last_index[n] = -1;
1899 }
1900
1901 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
1902                                int last_index)
1903 {
1904     int i;
1905     const int maxlevel = s->max_qcoeff;
1906     const int minlevel = s->min_qcoeff;
1907     int overflow = 0;
1908
1909     if (s->mb_intra) {
1910         i = 1; // skip clipping of intra dc
1911     } else
1912         i = 0;
1913
1914     for (; i <= last_index; i++) {
1915         const int j = s->intra_scantable.permutated[i];
1916         int level = block[j];
1917
1918         if (level > maxlevel) {
1919             level = maxlevel;
1920             overflow++;
1921         } else if (level < minlevel) {
1922             level = minlevel;
1923             overflow++;
1924         }
1925
1926         block[j] = level;
1927     }
1928
1929     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1930         av_log(s->avctx, AV_LOG_INFO,
1931                "warning, clipping %d dct coefficients to %d..%d\n",
1932                overflow, minlevel, maxlevel);
1933 }
1934
1935 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
1936 {
1937     int x, y;
1938     // FIXME optimize
1939     for (y = 0; y < 8; y++) {
1940         for (x = 0; x < 8; x++) {
1941             int x2, y2;
1942             int sum = 0;
1943             int sqr = 0;
1944             int count = 0;
1945
1946             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
1947                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
1948                     int v = ptr[x2 + y2 * stride];
1949                     sum += v;
1950                     sqr += v * v;
1951                     count++;
1952                 }
1953             }
1954             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
1955         }
1956     }
1957 }
1958
1959 static av_always_inline void encode_mb_internal(MpegEncContext *s,
1960                                                 int motion_x, int motion_y,
1961                                                 int mb_block_height,
1962                                                 int mb_block_width,
1963                                                 int mb_block_count)
1964 {
1965     int16_t weight[12][64];
1966     int16_t orig[12][64];
1967     const int mb_x = s->mb_x;
1968     const int mb_y = s->mb_y;
1969     int i;
1970     int skip_dct[12];
1971     int dct_offset = s->linesize * 8; // default for progressive frames
1972     int uv_dct_offset = s->uvlinesize * 8;
1973     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1974     ptrdiff_t wrap_y, wrap_c;
1975
1976     for (i = 0; i < mb_block_count; i++)
1977         skip_dct[i] = s->skipdct;
1978
1979     if (s->adaptive_quant) {
1980         const int last_qp = s->qscale;
1981         const int mb_xy = mb_x + mb_y * s->mb_stride;
1982
1983         s->lambda = s->lambda_table[mb_xy];
1984         update_qscale(s);
1985
1986         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
1987             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
1988             s->dquant = s->qscale - last_qp;
1989
1990             if (s->out_format == FMT_H263) {
1991                 s->dquant = av_clip(s->dquant, -2, 2);
1992
1993                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
1994                     if (!s->mb_intra) {
1995                         if (s->pict_type == AV_PICTURE_TYPE_B) {
1996                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
1997                                 s->dquant = 0;
1998                         }
1999                         if (s->mv_type == MV_TYPE_8X8)
2000                             s->dquant = 0;
2001                     }
2002                 }
2003             }
2004         }
2005         ff_set_qscale(s, last_qp + s->dquant);
2006     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
2007         ff_set_qscale(s, s->qscale + s->dquant);
2008
2009     wrap_y = s->linesize;
2010     wrap_c = s->uvlinesize;
2011     ptr_y  = s->new_picture.f->data[0] +
2012              (mb_y * 16 * wrap_y)              + mb_x * 16;
2013     ptr_cb = s->new_picture.f->data[1] +
2014              (mb_y * mb_block_height * wrap_c) + mb_x * mb_block_width;
2015     ptr_cr = s->new_picture.f->data[2] +
2016              (mb_y * mb_block_height * wrap_c) + mb_x * mb_block_width;
2017
2018     if((mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) && s->codec_id != AV_CODEC_ID_AMV){
2019         uint8_t *ebuf = s->edge_emu_buffer + 32;
2020         int cw = (s->width  + s->chroma_x_shift) >> s->chroma_x_shift;
2021         int ch = (s->height + s->chroma_y_shift) >> s->chroma_y_shift;
2022         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
2023                                  wrap_y, wrap_y,
2024                                  16, 16, mb_x * 16, mb_y * 16,
2025                                  s->width, s->height);
2026         ptr_y = ebuf;
2027         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y, ptr_cb,
2028                                  wrap_c, wrap_c,
2029                                  mb_block_width, mb_block_height,
2030                                  mb_x * mb_block_width, mb_y * mb_block_height,
2031                                  cw, ch);
2032         ptr_cb = ebuf + 18 * wrap_y;
2033         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y + 16, ptr_cr,
2034                                  wrap_c, wrap_c,
2035                                  mb_block_width, mb_block_height,
2036                                  mb_x * mb_block_width, mb_y * mb_block_height,
2037                                  cw, ch);
2038         ptr_cr = ebuf + 18 * wrap_y + 16;
2039     }
2040
2041     if (s->mb_intra) {
2042         if (s->flags & CODEC_FLAG_INTERLACED_DCT) {
2043             int progressive_score, interlaced_score;
2044
2045             s->interlaced_dct = 0;
2046             progressive_score = s->dsp.ildct_cmp[4](s, ptr_y,
2047                                                     NULL, wrap_y, 8) +
2048                                 s->dsp.ildct_cmp[4](s, ptr_y + wrap_y * 8,
2049                                                     NULL, wrap_y, 8) - 400;
2050
2051             if (progressive_score > 0) {
2052                 interlaced_score = s->dsp.ildct_cmp[4](s, ptr_y,
2053                                                        NULL, wrap_y * 2, 8) +
2054                                    s->dsp.ildct_cmp[4](s, ptr_y + wrap_y,
2055                                                        NULL, wrap_y * 2, 8);
2056                 if (progressive_score > interlaced_score) {
2057                     s->interlaced_dct = 1;
2058
2059                     dct_offset = wrap_y;
2060                     uv_dct_offset = wrap_c;
2061                     wrap_y <<= 1;
2062                     if (s->chroma_format == CHROMA_422 ||
2063                         s->chroma_format == CHROMA_444)
2064                         wrap_c <<= 1;
2065                 }
2066             }
2067         }
2068
2069         s->dsp.get_pixels(s->block[0], ptr_y                  , wrap_y);
2070         s->dsp.get_pixels(s->block[1], ptr_y              + 8 , wrap_y);
2071         s->dsp.get_pixels(s->block[2], ptr_y + dct_offset     , wrap_y);
2072         s->dsp.get_pixels(s->block[3], ptr_y + dct_offset + 8 , wrap_y);
2073
2074         if (s->flags & CODEC_FLAG_GRAY) {
2075             skip_dct[4] = 1;
2076             skip_dct[5] = 1;
2077         } else {
2078             s->dsp.get_pixels(s->block[4], ptr_cb, wrap_c);
2079             s->dsp.get_pixels(s->block[5], ptr_cr, wrap_c);
2080             if (!s->chroma_y_shift && s->chroma_x_shift) { /* 422 */
2081                 s->dsp.get_pixels(s->block[6], ptr_cb + uv_dct_offset, wrap_c);
2082                 s->dsp.get_pixels(s->block[7], ptr_cr + uv_dct_offset, wrap_c);
2083             } else if (!s->chroma_y_shift && !s->chroma_x_shift) { /* 444 */
2084                 s->dsp.get_pixels(s->block[6], ptr_cb + 8, wrap_c);
2085                 s->dsp.get_pixels(s->block[7], ptr_cr + 8, wrap_c);
2086                 s->dsp.get_pixels(s->block[8], ptr_cb + uv_dct_offset, wrap_c);
2087                 s->dsp.get_pixels(s->block[9], ptr_cr + uv_dct_offset, wrap_c);
2088                 s->dsp.get_pixels(s->block[10], ptr_cb + uv_dct_offset + 8, wrap_c);
2089                 s->dsp.get_pixels(s->block[11], ptr_cr + uv_dct_offset + 8, wrap_c);
2090             }
2091         }
2092     } else {
2093         op_pixels_func (*op_pix)[4];
2094         qpel_mc_func (*op_qpix)[16];
2095         uint8_t *dest_y, *dest_cb, *dest_cr;
2096
2097         dest_y  = s->dest[0];
2098         dest_cb = s->dest[1];
2099         dest_cr = s->dest[2];
2100
2101         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
2102             op_pix  = s->hdsp.put_pixels_tab;
2103             op_qpix = s->qdsp.put_qpel_pixels_tab;
2104         } else {
2105             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
2106             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
2107         }
2108
2109         if (s->mv_dir & MV_DIR_FORWARD) {
2110             ff_MPV_motion(s, dest_y, dest_cb, dest_cr, 0,
2111                           s->last_picture.f->data,
2112                           op_pix, op_qpix);
2113             op_pix  = s->hdsp.avg_pixels_tab;
2114             op_qpix = s->qdsp.avg_qpel_pixels_tab;
2115         }
2116         if (s->mv_dir & MV_DIR_BACKWARD) {
2117             ff_MPV_motion(s, dest_y, dest_cb, dest_cr, 1,
2118                           s->next_picture.f->data,
2119                           op_pix, op_qpix);
2120         }
2121
2122         if (s->flags & CODEC_FLAG_INTERLACED_DCT) {
2123             int progressive_score, interlaced_score;
2124
2125             s->interlaced_dct = 0;
2126             progressive_score = s->dsp.ildct_cmp[0](s, dest_y,
2127                                                     ptr_y,              wrap_y,
2128                                                     8) +
2129                                 s->dsp.ildct_cmp[0](s, dest_y + wrap_y * 8,
2130                                                     ptr_y + wrap_y * 8, wrap_y,
2131                                                     8) - 400;
2132
2133             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2134                 progressive_score -= 400;
2135
2136             if (progressive_score > 0) {
2137                 interlaced_score = s->dsp.ildct_cmp[0](s, dest_y,
2138                                                        ptr_y,
2139                                                        wrap_y * 2, 8) +
2140                                    s->dsp.ildct_cmp[0](s, dest_y + wrap_y,
2141                                                        ptr_y + wrap_y,
2142                                                        wrap_y * 2, 8);
2143
2144                 if (progressive_score > interlaced_score) {
2145                     s->interlaced_dct = 1;
2146
2147                     dct_offset = wrap_y;
2148                     uv_dct_offset = wrap_c;
2149                     wrap_y <<= 1;
2150                     if (s->chroma_format == CHROMA_422)
2151                         wrap_c <<= 1;
2152                 }
2153             }
2154         }
2155
2156         s->dsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2157         s->dsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2158         s->dsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2159                            dest_y + dct_offset, wrap_y);
2160         s->dsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2161                            dest_y + dct_offset + 8, wrap_y);
2162
2163         if (s->flags & CODEC_FLAG_GRAY) {
2164             skip_dct[4] = 1;
2165             skip_dct[5] = 1;
2166         } else {
2167             s->dsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2168             s->dsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2169             if (!s->chroma_y_shift) { /* 422 */
2170                 s->dsp.diff_pixels(s->block[6], ptr_cb + uv_dct_offset,
2171                                    dest_cb + uv_dct_offset, wrap_c);
2172                 s->dsp.diff_pixels(s->block[7], ptr_cr + uv_dct_offset,
2173                                    dest_cr + uv_dct_offset, wrap_c);
2174             }
2175         }
2176         /* pre quantization */
2177         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2178                 2 * s->qscale * s->qscale) {
2179             // FIXME optimize
2180             if (s->dsp.sad[1](NULL, ptr_y , dest_y,
2181                               wrap_y, 8) < 20 * s->qscale)
2182                 skip_dct[0] = 1;
2183             if (s->dsp.sad[1](NULL, ptr_y + 8,
2184                               dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2185                 skip_dct[1] = 1;
2186             if (s->dsp.sad[1](NULL, ptr_y + dct_offset,
2187                               dest_y + dct_offset, wrap_y, 8) < 20 * s->qscale)
2188                 skip_dct[2] = 1;
2189             if (s->dsp.sad[1](NULL, ptr_y + dct_offset + 8,
2190                               dest_y + dct_offset + 8,
2191                               wrap_y, 8) < 20 * s->qscale)
2192                 skip_dct[3] = 1;
2193             if (s->dsp.sad[1](NULL, ptr_cb, dest_cb,
2194                               wrap_c, 8) < 20 * s->qscale)
2195                 skip_dct[4] = 1;
2196             if (s->dsp.sad[1](NULL, ptr_cr, dest_cr,
2197                               wrap_c, 8) < 20 * s->qscale)
2198                 skip_dct[5] = 1;
2199             if (!s->chroma_y_shift) { /* 422 */
2200                 if (s->dsp.sad[1](NULL, ptr_cb + uv_dct_offset,
2201                                   dest_cb + uv_dct_offset,
2202                                   wrap_c, 8) < 20 * s->qscale)
2203                     skip_dct[6] = 1;
2204                 if (s->dsp.sad[1](NULL, ptr_cr + uv_dct_offset,
2205                                   dest_cr + uv_dct_offset,
2206                                   wrap_c, 8) < 20 * s->qscale)
2207                     skip_dct[7] = 1;
2208             }
2209         }
2210     }
2211
2212     if (s->quantizer_noise_shaping) {
2213         if (!skip_dct[0])
2214             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2215         if (!skip_dct[1])
2216             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2217         if (!skip_dct[2])
2218             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2219         if (!skip_dct[3])
2220             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2221         if (!skip_dct[4])
2222             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2223         if (!skip_dct[5])
2224             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2225         if (!s->chroma_y_shift) { /* 422 */
2226             if (!skip_dct[6])
2227                 get_visual_weight(weight[6], ptr_cb + uv_dct_offset,
2228                                   wrap_c);
2229             if (!skip_dct[7])
2230                 get_visual_weight(weight[7], ptr_cr + uv_dct_offset,
2231                                   wrap_c);
2232         }
2233         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2234     }
2235
2236     /* DCT & quantize */
2237     av_assert2(s->out_format != FMT_MJPEG || s->qscale == 8);
2238     {
2239         for (i = 0; i < mb_block_count; i++) {
2240             if (!skip_dct[i]) {
2241                 int overflow;
2242                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2243                 // FIXME we could decide to change to quantizer instead of
2244                 // clipping
2245                 // JS: I don't think that would be a good idea it could lower
2246                 //     quality instead of improve it. Just INTRADC clipping
2247                 //     deserves changes in quantizer
2248                 if (overflow)
2249                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2250             } else
2251                 s->block_last_index[i] = -1;
2252         }
2253         if (s->quantizer_noise_shaping) {
2254             for (i = 0; i < mb_block_count; i++) {
2255                 if (!skip_dct[i]) {
2256                     s->block_last_index[i] =
2257                         dct_quantize_refine(s, s->block[i], weight[i],
2258                                             orig[i], i, s->qscale);
2259                 }
2260             }
2261         }
2262
2263         if (s->luma_elim_threshold && !s->mb_intra)
2264             for (i = 0; i < 4; i++)
2265                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2266         if (s->chroma_elim_threshold && !s->mb_intra)
2267             for (i = 4; i < mb_block_count; i++)
2268                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2269
2270         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2271             for (i = 0; i < mb_block_count; i++) {
2272                 if (s->block_last_index[i] == -1)
2273                     s->coded_score[i] = INT_MAX / 256;
2274             }
2275         }
2276     }
2277
2278     if ((s->flags & CODEC_FLAG_GRAY) && s->mb_intra) {
2279         s->block_last_index[4] =
2280         s->block_last_index[5] = 0;
2281         s->block[4][0] =
2282         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2283         if (!s->chroma_y_shift) { /* 422 / 444 */
2284             for (i=6; i<12; i++) {
2285                 s->block_last_index[i] = 0;
2286                 s->block[i][0] = s->block[4][0];
2287             }
2288         }
2289     }
2290
2291     // non c quantize code returns incorrect block_last_index FIXME
2292     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2293         for (i = 0; i < mb_block_count; i++) {
2294             int j;
2295             if (s->block_last_index[i] > 0) {
2296                 for (j = 63; j > 0; j--) {
2297                     if (s->block[i][s->intra_scantable.permutated[j]])
2298                         break;
2299                 }
2300                 s->block_last_index[i] = j;
2301             }
2302         }
2303     }
2304
2305     /* huffman encode */
2306     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2307     case AV_CODEC_ID_MPEG1VIDEO:
2308     case AV_CODEC_ID_MPEG2VIDEO:
2309         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2310             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2311         break;
2312     case AV_CODEC_ID_MPEG4:
2313         if (CONFIG_MPEG4_ENCODER)
2314             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2315         break;
2316     case AV_CODEC_ID_MSMPEG4V2:
2317     case AV_CODEC_ID_MSMPEG4V3:
2318     case AV_CODEC_ID_WMV1:
2319         if (CONFIG_MSMPEG4_ENCODER)
2320             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2321         break;
2322     case AV_CODEC_ID_WMV2:
2323         if (CONFIG_WMV2_ENCODER)
2324             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2325         break;
2326     case AV_CODEC_ID_H261:
2327         if (CONFIG_H261_ENCODER)
2328             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2329         break;
2330     case AV_CODEC_ID_H263:
2331     case AV_CODEC_ID_H263P:
2332     case AV_CODEC_ID_FLV1:
2333     case AV_CODEC_ID_RV10:
2334     case AV_CODEC_ID_RV20:
2335         if (CONFIG_H263_ENCODER)
2336             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2337         break;
2338     case AV_CODEC_ID_MJPEG:
2339     case AV_CODEC_ID_AMV:
2340         if (CONFIG_MJPEG_ENCODER)
2341             ff_mjpeg_encode_mb(s, s->block);
2342         break;
2343     default:
2344         av_assert1(0);
2345     }
2346 }
2347
2348 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2349 {
2350     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 8, 6);
2351     else if (s->chroma_format == CHROMA_422) encode_mb_internal(s, motion_x, motion_y, 16, 8, 8);
2352     else encode_mb_internal(s, motion_x, motion_y, 16, 16, 12);
2353 }
2354
2355 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2356     int i;
2357
2358     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2359
2360     /* mpeg1 */
2361     d->mb_skip_run= s->mb_skip_run;
2362     for(i=0; i<3; i++)
2363         d->last_dc[i] = s->last_dc[i];
2364
2365     /* statistics */
2366     d->mv_bits= s->mv_bits;
2367     d->i_tex_bits= s->i_tex_bits;
2368     d->p_tex_bits= s->p_tex_bits;
2369     d->i_count= s->i_count;
2370     d->f_count= s->f_count;
2371     d->b_count= s->b_count;
2372     d->skip_count= s->skip_count;
2373     d->misc_bits= s->misc_bits;
2374     d->last_bits= 0;
2375
2376     d->mb_skipped= 0;
2377     d->qscale= s->qscale;
2378     d->dquant= s->dquant;
2379
2380     d->esc3_level_length= s->esc3_level_length;
2381 }
2382
2383 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2384     int i;
2385
2386     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2387     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2388
2389     /* mpeg1 */
2390     d->mb_skip_run= s->mb_skip_run;
2391     for(i=0; i<3; i++)
2392         d->last_dc[i] = s->last_dc[i];
2393
2394     /* statistics */
2395     d->mv_bits= s->mv_bits;
2396     d->i_tex_bits= s->i_tex_bits;
2397     d->p_tex_bits= s->p_tex_bits;
2398     d->i_count= s->i_count;
2399     d->f_count= s->f_count;
2400     d->b_count= s->b_count;
2401     d->skip_count= s->skip_count;
2402     d->misc_bits= s->misc_bits;
2403
2404     d->mb_intra= s->mb_intra;
2405     d->mb_skipped= s->mb_skipped;
2406     d->mv_type= s->mv_type;
2407     d->mv_dir= s->mv_dir;
2408     d->pb= s->pb;
2409     if(s->data_partitioning){
2410         d->pb2= s->pb2;
2411         d->tex_pb= s->tex_pb;
2412     }
2413     d->block= s->block;
2414     for(i=0; i<8; i++)
2415         d->block_last_index[i]= s->block_last_index[i];
2416     d->interlaced_dct= s->interlaced_dct;
2417     d->qscale= s->qscale;
2418
2419     d->esc3_level_length= s->esc3_level_length;
2420 }
2421
2422 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2423                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2424                            int *dmin, int *next_block, int motion_x, int motion_y)
2425 {
2426     int score;
2427     uint8_t *dest_backup[3];
2428
2429     copy_context_before_encode(s, backup, type);
2430
2431     s->block= s->blocks[*next_block];
2432     s->pb= pb[*next_block];
2433     if(s->data_partitioning){
2434         s->pb2   = pb2   [*next_block];
2435         s->tex_pb= tex_pb[*next_block];
2436     }
2437
2438     if(*next_block){
2439         memcpy(dest_backup, s->dest, sizeof(s->dest));
2440         s->dest[0] = s->rd_scratchpad;
2441         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
2442         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
2443         av_assert0(s->linesize >= 32); //FIXME
2444     }
2445
2446     encode_mb(s, motion_x, motion_y);
2447
2448     score= put_bits_count(&s->pb);
2449     if(s->data_partitioning){
2450         score+= put_bits_count(&s->pb2);
2451         score+= put_bits_count(&s->tex_pb);
2452     }
2453
2454     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2455         ff_MPV_decode_mb(s, s->block);
2456
2457         score *= s->lambda2;
2458         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2459     }
2460
2461     if(*next_block){
2462         memcpy(s->dest, dest_backup, sizeof(s->dest));
2463     }
2464
2465     if(score<*dmin){
2466         *dmin= score;
2467         *next_block^=1;
2468
2469         copy_context_after_encode(best, s, type);
2470     }
2471 }
2472
2473 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2474     uint32_t *sq = ff_square_tab + 256;
2475     int acc=0;
2476     int x,y;
2477
2478     if(w==16 && h==16)
2479         return s->dsp.sse[0](NULL, src1, src2, stride, 16);
2480     else if(w==8 && h==8)
2481         return s->dsp.sse[1](NULL, src1, src2, stride, 8);
2482
2483     for(y=0; y<h; y++){
2484         for(x=0; x<w; x++){
2485             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2486         }
2487     }
2488
2489     av_assert2(acc>=0);
2490
2491     return acc;
2492 }
2493
2494 static int sse_mb(MpegEncContext *s){
2495     int w= 16;
2496     int h= 16;
2497
2498     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2499     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2500
2501     if(w==16 && h==16)
2502       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2503         return  s->dsp.nsse[0](s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2504                +s->dsp.nsse[1](s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2505                +s->dsp.nsse[1](s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2506       }else{
2507         return  s->dsp.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2508                +s->dsp.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2509                +s->dsp.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2510       }
2511     else
2512         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2513                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2514                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2515 }
2516
2517 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2518     MpegEncContext *s= *(void**)arg;
2519
2520
2521     s->me.pre_pass=1;
2522     s->me.dia_size= s->avctx->pre_dia_size;
2523     s->first_slice_line=1;
2524     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2525         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2526             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2527         }
2528         s->first_slice_line=0;
2529     }
2530
2531     s->me.pre_pass=0;
2532
2533     return 0;
2534 }
2535
2536 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2537     MpegEncContext *s= *(void**)arg;
2538
2539     ff_check_alignment();
2540
2541     s->me.dia_size= s->avctx->dia_size;
2542     s->first_slice_line=1;
2543     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2544         s->mb_x=0; //for block init below
2545         ff_init_block_index(s);
2546         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2547             s->block_index[0]+=2;
2548             s->block_index[1]+=2;
2549             s->block_index[2]+=2;
2550             s->block_index[3]+=2;
2551
2552             /* compute motion vector & mb_type and store in context */
2553             if(s->pict_type==AV_PICTURE_TYPE_B)
2554                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2555             else
2556                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2557         }
2558         s->first_slice_line=0;
2559     }
2560     return 0;
2561 }
2562
2563 static int mb_var_thread(AVCodecContext *c, void *arg){
2564     MpegEncContext *s= *(void**)arg;
2565     int mb_x, mb_y;
2566
2567     ff_check_alignment();
2568
2569     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2570         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2571             int xx = mb_x * 16;
2572             int yy = mb_y * 16;
2573             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2574             int varc;
2575             int sum = s->dsp.pix_sum(pix, s->linesize);
2576
2577             varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)sum*sum)>>8) + 500 + 128)>>8;
2578
2579             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2580             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2581             s->me.mb_var_sum_temp    += varc;
2582         }
2583     }
2584     return 0;
2585 }
2586
2587 static void write_slice_end(MpegEncContext *s){
2588     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2589         if(s->partitioned_frame){
2590             ff_mpeg4_merge_partitions(s);
2591         }
2592
2593         ff_mpeg4_stuffing(&s->pb);
2594     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2595         ff_mjpeg_encode_stuffing(s);
2596     }
2597
2598     avpriv_align_put_bits(&s->pb);
2599     flush_put_bits(&s->pb);
2600
2601     if((s->flags&CODEC_FLAG_PASS1) && !s->partitioned_frame)
2602         s->misc_bits+= get_bits_diff(s);
2603 }
2604
2605 static void write_mb_info(MpegEncContext *s)
2606 {
2607     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2608     int offset = put_bits_count(&s->pb);
2609     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2610     int gobn = s->mb_y / s->gob_index;
2611     int pred_x, pred_y;
2612     if (CONFIG_H263_ENCODER)
2613         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2614     bytestream_put_le32(&ptr, offset);
2615     bytestream_put_byte(&ptr, s->qscale);
2616     bytestream_put_byte(&ptr, gobn);
2617     bytestream_put_le16(&ptr, mba);
2618     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2619     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2620     /* 4MV not implemented */
2621     bytestream_put_byte(&ptr, 0); /* hmv2 */
2622     bytestream_put_byte(&ptr, 0); /* vmv2 */
2623 }
2624
2625 static void update_mb_info(MpegEncContext *s, int startcode)
2626 {
2627     if (!s->mb_info)
2628         return;
2629     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2630         s->mb_info_size += 12;
2631         s->prev_mb_info = s->last_mb_info;
2632     }
2633     if (startcode) {
2634         s->prev_mb_info = put_bits_count(&s->pb)/8;
2635         /* This might have incremented mb_info_size above, and we return without
2636          * actually writing any info into that slot yet. But in that case,
2637          * this will be called again at the start of the after writing the
2638          * start code, actually writing the mb info. */
2639         return;
2640     }
2641
2642     s->last_mb_info = put_bits_count(&s->pb)/8;
2643     if (!s->mb_info_size)
2644         s->mb_info_size += 12;
2645     write_mb_info(s);
2646 }
2647
2648 static int encode_thread(AVCodecContext *c, void *arg){
2649     MpegEncContext *s= *(void**)arg;
2650     int mb_x, mb_y, pdif = 0;
2651     int chr_h= 16>>s->chroma_y_shift;
2652     int i, j;
2653     MpegEncContext best_s, backup_s;
2654     uint8_t bit_buf[2][MAX_MB_BYTES];
2655     uint8_t bit_buf2[2][MAX_MB_BYTES];
2656     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2657     PutBitContext pb[2], pb2[2], tex_pb[2];
2658
2659     ff_check_alignment();
2660
2661     for(i=0; i<2; i++){
2662         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2663         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2664         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2665     }
2666
2667     s->last_bits= put_bits_count(&s->pb);
2668     s->mv_bits=0;
2669     s->misc_bits=0;
2670     s->i_tex_bits=0;
2671     s->p_tex_bits=0;
2672     s->i_count=0;
2673     s->f_count=0;
2674     s->b_count=0;
2675     s->skip_count=0;
2676
2677     for(i=0; i<3; i++){
2678         /* init last dc values */
2679         /* note: quant matrix value (8) is implied here */
2680         s->last_dc[i] = 128 << s->intra_dc_precision;
2681
2682         s->current_picture.f->error[i] = 0;
2683     }
2684     if(s->codec_id==AV_CODEC_ID_AMV){
2685         s->last_dc[0] = 128*8/13;
2686         s->last_dc[1] = 128*8/14;
2687         s->last_dc[2] = 128*8/14;
2688     }
2689     s->mb_skip_run = 0;
2690     memset(s->last_mv, 0, sizeof(s->last_mv));
2691
2692     s->last_mv_dir = 0;
2693
2694     switch(s->codec_id){
2695     case AV_CODEC_ID_H263:
2696     case AV_CODEC_ID_H263P:
2697     case AV_CODEC_ID_FLV1:
2698         if (CONFIG_H263_ENCODER)
2699             s->gob_index = ff_h263_get_gob_height(s);
2700         break;
2701     case AV_CODEC_ID_MPEG4:
2702         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2703             ff_mpeg4_init_partitions(s);
2704         break;
2705     }
2706
2707     s->resync_mb_x=0;
2708     s->resync_mb_y=0;
2709     s->first_slice_line = 1;
2710     s->ptr_lastgob = s->pb.buf;
2711     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2712         s->mb_x=0;
2713         s->mb_y= mb_y;
2714
2715         ff_set_qscale(s, s->qscale);
2716         ff_init_block_index(s);
2717
2718         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2719             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2720             int mb_type= s->mb_type[xy];
2721 //            int d;
2722             int dmin= INT_MAX;
2723             int dir;
2724
2725             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2726                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2727                 return -1;
2728             }
2729             if(s->data_partitioning){
2730                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2731                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2732                     av_log(s->avctx, AV_LOG_ERROR, "encoded partitioned frame too large\n");
2733                     return -1;
2734                 }
2735             }
2736
2737             s->mb_x = mb_x;
2738             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2739             ff_update_block_index(s);
2740
2741             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
2742                 ff_h261_reorder_mb_index(s);
2743                 xy= s->mb_y*s->mb_stride + s->mb_x;
2744                 mb_type= s->mb_type[xy];
2745             }
2746
2747             /* write gob / video packet header  */
2748             if(s->rtp_mode){
2749                 int current_packet_size, is_gob_start;
2750
2751                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2752
2753                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2754
2755                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2756
2757                 switch(s->codec_id){
2758                 case AV_CODEC_ID_H263:
2759                 case AV_CODEC_ID_H263P:
2760                     if(!s->h263_slice_structured)
2761                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2762                     break;
2763                 case AV_CODEC_ID_MPEG2VIDEO:
2764                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2765                 case AV_CODEC_ID_MPEG1VIDEO:
2766                     if(s->mb_skip_run) is_gob_start=0;
2767                     break;
2768                 case AV_CODEC_ID_MJPEG:
2769                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2770                     break;
2771                 }
2772
2773                 if(is_gob_start){
2774                     if(s->start_mb_y != mb_y || mb_x!=0){
2775                         write_slice_end(s);
2776
2777                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
2778                             ff_mpeg4_init_partitions(s);
2779                         }
2780                     }
2781
2782                     av_assert2((put_bits_count(&s->pb)&7) == 0);
2783                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2784
2785                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
2786                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2787                         int d = 100 / s->error_rate;
2788                         if(r % d == 0){
2789                             current_packet_size=0;
2790                             s->pb.buf_ptr= s->ptr_lastgob;
2791                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2792                         }
2793                     }
2794
2795                     if (s->avctx->rtp_callback){
2796                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2797                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2798                     }
2799                     update_mb_info(s, 1);
2800
2801                     switch(s->codec_id){
2802                     case AV_CODEC_ID_MPEG4:
2803                         if (CONFIG_MPEG4_ENCODER) {
2804                             ff_mpeg4_encode_video_packet_header(s);
2805                             ff_mpeg4_clean_buffers(s);
2806                         }
2807                     break;
2808                     case AV_CODEC_ID_MPEG1VIDEO:
2809                     case AV_CODEC_ID_MPEG2VIDEO:
2810                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2811                             ff_mpeg1_encode_slice_header(s);
2812                             ff_mpeg1_clean_buffers(s);
2813                         }
2814                     break;
2815                     case AV_CODEC_ID_H263:
2816                     case AV_CODEC_ID_H263P:
2817                         if (CONFIG_H263_ENCODER)
2818                             ff_h263_encode_gob_header(s, mb_y);
2819                     break;
2820                     }
2821
2822                     if(s->flags&CODEC_FLAG_PASS1){
2823                         int bits= put_bits_count(&s->pb);
2824                         s->misc_bits+= bits - s->last_bits;
2825                         s->last_bits= bits;
2826                     }
2827
2828                     s->ptr_lastgob += current_packet_size;
2829                     s->first_slice_line=1;
2830                     s->resync_mb_x=mb_x;
2831                     s->resync_mb_y=mb_y;
2832                 }
2833             }
2834
2835             if(  (s->resync_mb_x   == s->mb_x)
2836                && s->resync_mb_y+1 == s->mb_y){
2837                 s->first_slice_line=0;
2838             }
2839
2840             s->mb_skipped=0;
2841             s->dquant=0; //only for QP_RD
2842
2843             update_mb_info(s, 0);
2844
2845             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
2846                 int next_block=0;
2847                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2848
2849                 copy_context_before_encode(&backup_s, s, -1);
2850                 backup_s.pb= s->pb;
2851                 best_s.data_partitioning= s->data_partitioning;
2852                 best_s.partitioned_frame= s->partitioned_frame;
2853                 if(s->data_partitioning){
2854                     backup_s.pb2= s->pb2;
2855                     backup_s.tex_pb= s->tex_pb;
2856                 }
2857
2858                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2859                     s->mv_dir = MV_DIR_FORWARD;
2860                     s->mv_type = MV_TYPE_16X16;
2861                     s->mb_intra= 0;
2862                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2863                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2864                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2865                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2866                 }
2867                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2868                     s->mv_dir = MV_DIR_FORWARD;
2869                     s->mv_type = MV_TYPE_FIELD;
2870                     s->mb_intra= 0;
2871                     for(i=0; i<2; i++){
2872                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2873                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2874                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2875                     }
2876                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2877                                  &dmin, &next_block, 0, 0);
2878                 }
2879                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2880                     s->mv_dir = MV_DIR_FORWARD;
2881                     s->mv_type = MV_TYPE_16X16;
2882                     s->mb_intra= 0;
2883                     s->mv[0][0][0] = 0;
2884                     s->mv[0][0][1] = 0;
2885                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2886                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2887                 }
2888                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2889                     s->mv_dir = MV_DIR_FORWARD;
2890                     s->mv_type = MV_TYPE_8X8;
2891                     s->mb_intra= 0;
2892                     for(i=0; i<4; i++){
2893                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2894                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2895                     }
2896                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2897                                  &dmin, &next_block, 0, 0);
2898                 }
2899                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2900                     s->mv_dir = MV_DIR_FORWARD;
2901                     s->mv_type = MV_TYPE_16X16;
2902                     s->mb_intra= 0;
2903                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2904                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2905                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2906                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2907                 }
2908                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2909                     s->mv_dir = MV_DIR_BACKWARD;
2910                     s->mv_type = MV_TYPE_16X16;
2911                     s->mb_intra= 0;
2912                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2913                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2914                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2915                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2916                 }
2917                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2918                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2919                     s->mv_type = MV_TYPE_16X16;
2920                     s->mb_intra= 0;
2921                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2922                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2923                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2924                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2925                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2926                                  &dmin, &next_block, 0, 0);
2927                 }
2928                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2929                     s->mv_dir = MV_DIR_FORWARD;
2930                     s->mv_type = MV_TYPE_FIELD;
2931                     s->mb_intra= 0;
2932                     for(i=0; i<2; i++){
2933                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2934                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2935                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2936                     }
2937                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2938                                  &dmin, &next_block, 0, 0);
2939                 }
2940                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2941                     s->mv_dir = MV_DIR_BACKWARD;
2942                     s->mv_type = MV_TYPE_FIELD;
2943                     s->mb_intra= 0;
2944                     for(i=0; i<2; i++){
2945                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2946                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2947                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2948                     }
2949                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2950                                  &dmin, &next_block, 0, 0);
2951                 }
2952                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2953                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2954                     s->mv_type = MV_TYPE_FIELD;
2955                     s->mb_intra= 0;
2956                     for(dir=0; dir<2; dir++){
2957                         for(i=0; i<2; i++){
2958                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2959                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2960                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2961                         }
2962                     }
2963                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2964                                  &dmin, &next_block, 0, 0);
2965                 }
2966                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2967                     s->mv_dir = 0;
2968                     s->mv_type = MV_TYPE_16X16;
2969                     s->mb_intra= 1;
2970                     s->mv[0][0][0] = 0;
2971                     s->mv[0][0][1] = 0;
2972                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2973                                  &dmin, &next_block, 0, 0);
2974                     if(s->h263_pred || s->h263_aic){
2975                         if(best_s.mb_intra)
2976                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2977                         else
2978                             ff_clean_intra_table_entries(s); //old mode?
2979                     }
2980                 }
2981
2982                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
2983                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2984                         const int last_qp= backup_s.qscale;
2985                         int qpi, qp, dc[6];
2986                         int16_t ac[6][16];
2987                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2988                         static const int dquant_tab[4]={-1,1,-2,2};
2989                         int storecoefs = s->mb_intra && s->dc_val[0];
2990
2991                         av_assert2(backup_s.dquant == 0);
2992
2993                         //FIXME intra
2994                         s->mv_dir= best_s.mv_dir;
2995                         s->mv_type = MV_TYPE_16X16;
2996                         s->mb_intra= best_s.mb_intra;
2997                         s->mv[0][0][0] = best_s.mv[0][0][0];
2998                         s->mv[0][0][1] = best_s.mv[0][0][1];
2999                         s->mv[1][0][0] = best_s.mv[1][0][0];
3000                         s->mv[1][0][1] = best_s.mv[1][0][1];
3001
3002                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
3003                         for(; qpi<4; qpi++){
3004                             int dquant= dquant_tab[qpi];
3005                             qp= last_qp + dquant;
3006                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
3007                                 continue;
3008                             backup_s.dquant= dquant;
3009                             if(storecoefs){
3010                                 for(i=0; i<6; i++){
3011                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
3012                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(int16_t)*16);
3013                                 }
3014                             }
3015
3016                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3017                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
3018                             if(best_s.qscale != qp){
3019                                 if(storecoefs){
3020                                     for(i=0; i<6; i++){
3021                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
3022                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(int16_t)*16);
3023                                     }
3024                                 }
3025                             }
3026                         }
3027                     }
3028                 }
3029                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
3030                     int mx= s->b_direct_mv_table[xy][0];
3031                     int my= s->b_direct_mv_table[xy][1];
3032
3033                     backup_s.dquant = 0;
3034                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3035                     s->mb_intra= 0;
3036                     ff_mpeg4_set_direct_mv(s, mx, my);
3037                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3038                                  &dmin, &next_block, mx, my);
3039                 }
3040                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
3041                     backup_s.dquant = 0;
3042                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3043                     s->mb_intra= 0;
3044                     ff_mpeg4_set_direct_mv(s, 0, 0);
3045                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3046                                  &dmin, &next_block, 0, 0);
3047                 }
3048                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
3049                     int coded=0;
3050                     for(i=0; i<6; i++)
3051                         coded |= s->block_last_index[i];
3052                     if(coded){
3053                         int mx,my;
3054                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
3055                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
3056                             mx=my=0; //FIXME find the one we actually used
3057                             ff_mpeg4_set_direct_mv(s, mx, my);
3058                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
3059                             mx= s->mv[1][0][0];
3060                             my= s->mv[1][0][1];
3061                         }else{
3062                             mx= s->mv[0][0][0];
3063                             my= s->mv[0][0][1];
3064                         }
3065
3066                         s->mv_dir= best_s.mv_dir;
3067                         s->mv_type = best_s.mv_type;
3068                         s->mb_intra= 0;
3069 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
3070                         s->mv[0][0][1] = best_s.mv[0][0][1];
3071                         s->mv[1][0][0] = best_s.mv[1][0][0];
3072                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
3073                         backup_s.dquant= 0;
3074                         s->skipdct=1;
3075                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3076                                         &dmin, &next_block, mx, my);
3077                         s->skipdct=0;
3078                     }
3079                 }
3080
3081                 s->current_picture.qscale_table[xy] = best_s.qscale;
3082
3083                 copy_context_after_encode(s, &best_s, -1);
3084
3085                 pb_bits_count= put_bits_count(&s->pb);
3086                 flush_put_bits(&s->pb);
3087                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
3088                 s->pb= backup_s.pb;
3089
3090                 if(s->data_partitioning){
3091                     pb2_bits_count= put_bits_count(&s->pb2);
3092                     flush_put_bits(&s->pb2);
3093                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
3094                     s->pb2= backup_s.pb2;
3095
3096                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
3097                     flush_put_bits(&s->tex_pb);
3098                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
3099                     s->tex_pb= backup_s.tex_pb;
3100                 }
3101                 s->last_bits= put_bits_count(&s->pb);
3102
3103                 if (CONFIG_H263_ENCODER &&
3104                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3105                     ff_h263_update_motion_val(s);
3106
3107                 if(next_block==0){ //FIXME 16 vs linesize16
3108                     s->hdsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
3109                     s->hdsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
3110                     s->hdsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
3111                 }
3112
3113                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
3114                     ff_MPV_decode_mb(s, s->block);
3115             } else {
3116                 int motion_x = 0, motion_y = 0;
3117                 s->mv_type=MV_TYPE_16X16;
3118                 // only one MB-Type possible
3119
3120                 switch(mb_type){
3121                 case CANDIDATE_MB_TYPE_INTRA:
3122                     s->mv_dir = 0;
3123                     s->mb_intra= 1;
3124                     motion_x= s->mv[0][0][0] = 0;
3125                     motion_y= s->mv[0][0][1] = 0;
3126                     break;
3127                 case CANDIDATE_MB_TYPE_INTER:
3128                     s->mv_dir = MV_DIR_FORWARD;
3129                     s->mb_intra= 0;
3130                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
3131                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
3132                     break;
3133                 case CANDIDATE_MB_TYPE_INTER_I:
3134                     s->mv_dir = MV_DIR_FORWARD;
3135                     s->mv_type = MV_TYPE_FIELD;
3136                     s->mb_intra= 0;
3137                     for(i=0; i<2; i++){
3138                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3139                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3140                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3141                     }
3142                     break;
3143                 case CANDIDATE_MB_TYPE_INTER4V:
3144                     s->mv_dir = MV_DIR_FORWARD;
3145                     s->mv_type = MV_TYPE_8X8;
3146                     s->mb_intra= 0;
3147                     for(i=0; i<4; i++){
3148                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3149                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3150                     }
3151                     break;
3152                 case CANDIDATE_MB_TYPE_DIRECT:
3153                     if (CONFIG_MPEG4_ENCODER) {
3154                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3155                         s->mb_intra= 0;
3156                         motion_x=s->b_direct_mv_table[xy][0];
3157                         motion_y=s->b_direct_mv_table[xy][1];
3158                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
3159                     }
3160                     break;
3161                 case CANDIDATE_MB_TYPE_DIRECT0:
3162                     if (CONFIG_MPEG4_ENCODER) {
3163                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3164                         s->mb_intra= 0;
3165                         ff_mpeg4_set_direct_mv(s, 0, 0);
3166                     }
3167                     break;
3168                 case CANDIDATE_MB_TYPE_BIDIR:
3169                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3170                     s->mb_intra= 0;
3171                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3172                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3173                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3174                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3175                     break;
3176                 case CANDIDATE_MB_TYPE_BACKWARD:
3177                     s->mv_dir = MV_DIR_BACKWARD;
3178                     s->mb_intra= 0;
3179                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3180                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3181                     break;
3182                 case CANDIDATE_MB_TYPE_FORWARD:
3183                     s->mv_dir = MV_DIR_FORWARD;
3184                     s->mb_intra= 0;
3185                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3186                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3187                     break;
3188                 case CANDIDATE_MB_TYPE_FORWARD_I:
3189                     s->mv_dir = MV_DIR_FORWARD;
3190                     s->mv_type = MV_TYPE_FIELD;
3191                     s->mb_intra= 0;
3192                     for(i=0; i<2; i++){
3193                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3194                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3195                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3196                     }
3197                     break;
3198                 case CANDIDATE_MB_TYPE_BACKWARD_I:
3199                     s->mv_dir = MV_DIR_BACKWARD;
3200                     s->mv_type = MV_TYPE_FIELD;
3201                     s->mb_intra= 0;
3202                     for(i=0; i<2; i++){
3203                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3204                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3205                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3206                     }
3207                     break;
3208                 case CANDIDATE_MB_TYPE_BIDIR_I:
3209                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3210                     s->mv_type = MV_TYPE_FIELD;
3211                     s->mb_intra= 0;
3212                     for(dir=0; dir<2; dir++){
3213                         for(i=0; i<2; i++){
3214                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3215                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3216                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3217                         }
3218                     }
3219                     break;
3220                 default:
3221                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
3222                 }
3223
3224                 encode_mb(s, motion_x, motion_y);
3225
3226                 // RAL: Update last macroblock type
3227                 s->last_mv_dir = s->mv_dir;
3228
3229                 if (CONFIG_H263_ENCODER &&
3230                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3231                     ff_h263_update_motion_val(s);
3232
3233                 ff_MPV_decode_mb(s, s->block);
3234             }
3235
3236             /* clean the MV table in IPS frames for direct mode in B frames */
3237             if(s->mb_intra /* && I,P,S_TYPE */){
3238                 s->p_mv_table[xy][0]=0;
3239                 s->p_mv_table[xy][1]=0;
3240             }
3241
3242             if(s->flags&CODEC_FLAG_PSNR){
3243                 int w= 16;
3244                 int h= 16;
3245
3246                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
3247                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
3248
3249                 s->current_picture.f->error[0] += sse(
3250                     s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
3251                     s->dest[0], w, h, s->linesize);
3252                 s->current_picture.f->error[1] += sse(
3253                     s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3254                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3255                 s->current_picture.f->error[2] += sse(
3256                     s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3257                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3258             }
3259             if(s->loop_filter){
3260                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
3261                     ff_h263_loop_filter(s);
3262             }
3263             av_dlog(s->avctx, "MB %d %d bits\n",
3264                     s->mb_x + s->mb_y * s->mb_stride, put_bits_count(&s->pb));
3265         }
3266     }
3267
3268     //not beautiful here but we must write it before flushing so it has to be here
3269     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
3270         ff_msmpeg4_encode_ext_header(s);
3271
3272     write_slice_end(s);
3273
3274     /* Send the last GOB if RTP */
3275     if (s->avctx->rtp_callback) {
3276         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
3277         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
3278         /* Call the RTP callback to send the last GOB */
3279         emms_c();
3280         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
3281     }
3282
3283     return 0;
3284 }
3285
3286 #define MERGE(field) dst->field += src->field; src->field=0
3287 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
3288     MERGE(me.scene_change_score);
3289     MERGE(me.mc_mb_var_sum_temp);
3290     MERGE(me.mb_var_sum_temp);
3291 }
3292
3293 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
3294     int i;
3295
3296     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
3297     MERGE(dct_count[1]);
3298     MERGE(mv_bits);
3299     MERGE(i_tex_bits);
3300     MERGE(p_tex_bits);
3301     MERGE(i_count);
3302     MERGE(f_count);
3303     MERGE(b_count);
3304     MERGE(skip_count);
3305     MERGE(misc_bits);
3306     MERGE(er.error_count);
3307     MERGE(padding_bug_score);
3308     MERGE(current_picture.f->error[0]);
3309     MERGE(current_picture.f->error[1]);
3310     MERGE(current_picture.f->error[2]);
3311
3312     if(dst->avctx->noise_reduction){
3313         for(i=0; i<64; i++){
3314             MERGE(dct_error_sum[0][i]);
3315             MERGE(dct_error_sum[1][i]);
3316         }
3317     }
3318
3319     assert(put_bits_count(&src->pb) % 8 ==0);
3320     assert(put_bits_count(&dst->pb) % 8 ==0);
3321     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3322     flush_put_bits(&dst->pb);
3323 }
3324
3325 static int estimate_qp(MpegEncContext *s, int dry_run){
3326     if (s->next_lambda){
3327         s->current_picture_ptr->f->quality =
3328         s->current_picture.f->quality = s->next_lambda;
3329         if(!dry_run) s->next_lambda= 0;
3330     } else if (!s->fixed_qscale) {
3331         s->current_picture_ptr->f->quality =
3332         s->current_picture.f->quality = ff_rate_estimate_qscale(s, dry_run);
3333         if (s->current_picture.f->quality < 0)
3334             return -1;
3335     }
3336
3337     if(s->adaptive_quant){
3338         switch(s->codec_id){
3339         case AV_CODEC_ID_MPEG4:
3340             if (CONFIG_MPEG4_ENCODER)
3341                 ff_clean_mpeg4_qscales(s);
3342             break;
3343         case AV_CODEC_ID_H263:
3344         case AV_CODEC_ID_H263P:
3345         case AV_CODEC_ID_FLV1:
3346             if (CONFIG_H263_ENCODER)
3347                 ff_clean_h263_qscales(s);
3348             break;
3349         default:
3350             ff_init_qscale_tab(s);
3351         }
3352
3353         s->lambda= s->lambda_table[0];
3354         //FIXME broken
3355     }else
3356         s->lambda = s->current_picture.f->quality;
3357     update_qscale(s);
3358     return 0;
3359 }
3360
3361 /* must be called before writing the header */
3362 static void set_frame_distances(MpegEncContext * s){
3363     av_assert1(s->current_picture_ptr->f->pts != AV_NOPTS_VALUE);
3364     s->time = s->current_picture_ptr->f->pts * s->avctx->time_base.num;
3365
3366     if(s->pict_type==AV_PICTURE_TYPE_B){
3367         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3368         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3369     }else{
3370         s->pp_time= s->time - s->last_non_b_time;
3371         s->last_non_b_time= s->time;
3372         assert(s->picture_number==0 || s->pp_time > 0);
3373     }
3374 }
3375
3376 static int encode_picture(MpegEncContext *s, int picture_number)
3377 {
3378     int i, ret;
3379     int bits;
3380     int context_count = s->slice_context_count;
3381
3382     s->picture_number = picture_number;
3383
3384     /* Reset the average MB variance */
3385     s->me.mb_var_sum_temp    =
3386     s->me.mc_mb_var_sum_temp = 0;
3387
3388     /* we need to initialize some time vars before we can encode b-frames */
3389     // RAL: Condition added for MPEG1VIDEO
3390     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3391         set_frame_distances(s);
3392     if(CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4)
3393         ff_set_mpeg4_time(s);
3394
3395     s->me.scene_change_score=0;
3396
3397 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3398
3399     if(s->pict_type==AV_PICTURE_TYPE_I){
3400         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3401         else                        s->no_rounding=0;
3402     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3403         if(s->flipflop_rounding || s->codec_id == AV_CODEC_ID_H263P || s->codec_id == AV_CODEC_ID_MPEG4)
3404             s->no_rounding ^= 1;
3405     }
3406
3407     if(s->flags & CODEC_FLAG_PASS2){
3408         if (estimate_qp(s,1) < 0)
3409             return -1;
3410         ff_get_2pass_fcode(s);
3411     }else if(!(s->flags & CODEC_FLAG_QSCALE)){
3412         if(s->pict_type==AV_PICTURE_TYPE_B)
3413             s->lambda= s->last_lambda_for[s->pict_type];
3414         else
3415             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3416         update_qscale(s);
3417     }
3418
3419     if(s->codec_id != AV_CODEC_ID_AMV && s->codec_id != AV_CODEC_ID_MJPEG){
3420         if(s->q_chroma_intra_matrix   != s->q_intra_matrix  ) av_freep(&s->q_chroma_intra_matrix);
3421         if(s->q_chroma_intra_matrix16 != s->q_intra_matrix16) av_freep(&s->q_chroma_intra_matrix16);
3422         s->q_chroma_intra_matrix   = s->q_intra_matrix;
3423         s->q_chroma_intra_matrix16 = s->q_intra_matrix16;
3424     }
3425
3426     s->mb_intra=0; //for the rate distortion & bit compare functions
3427     for(i=1; i<context_count; i++){
3428         ret = ff_update_duplicate_context(s->thread_context[i], s);
3429         if (ret < 0)
3430             return ret;
3431     }
3432
3433     if(ff_init_me(s)<0)
3434         return -1;
3435
3436     /* Estimate motion for every MB */
3437     if(s->pict_type != AV_PICTURE_TYPE_I){
3438         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
3439         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
3440         if (s->pict_type != AV_PICTURE_TYPE_B) {
3441             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
3442                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3443             }
3444         }
3445
3446         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3447     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3448         /* I-Frame */
3449         for(i=0; i<s->mb_stride*s->mb_height; i++)
3450             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3451
3452         if(!s->fixed_qscale){
3453             /* finding spatial complexity for I-frame rate control */
3454             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3455         }
3456     }
3457     for(i=1; i<context_count; i++){
3458         merge_context_after_me(s, s->thread_context[i]);
3459     }
3460     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3461     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3462     emms_c();
3463
3464     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
3465         s->pict_type= AV_PICTURE_TYPE_I;
3466         for(i=0; i<s->mb_stride*s->mb_height; i++)
3467             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3468         if(s->msmpeg4_version >= 3)
3469             s->no_rounding=1;
3470         av_dlog(s, "Scene change detected, encoding as I Frame %"PRId64" %"PRId64"\n",
3471                 s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3472     }
3473
3474     if(!s->umvplus){
3475         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3476             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3477
3478             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3479                 int a,b;
3480                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3481                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3482                 s->f_code= FFMAX3(s->f_code, a, b);
3483             }
3484
3485             ff_fix_long_p_mvs(s);
3486             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3487             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3488                 int j;
3489                 for(i=0; i<2; i++){
3490                     for(j=0; j<2; j++)
3491                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3492                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3493                 }
3494             }
3495         }
3496
3497         if(s->pict_type==AV_PICTURE_TYPE_B){
3498             int a, b;
3499
3500             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3501             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3502             s->f_code = FFMAX(a, b);
3503
3504             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3505             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3506             s->b_code = FFMAX(a, b);
3507
3508             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3509             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3510             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3511             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3512             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3513                 int dir, j;
3514                 for(dir=0; dir<2; dir++){
3515                     for(i=0; i<2; i++){
3516                         for(j=0; j<2; j++){
3517                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3518                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3519                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3520                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3521                         }
3522                     }
3523                 }
3524             }
3525         }
3526     }
3527
3528     if (estimate_qp(s, 0) < 0)
3529         return -1;
3530
3531     if(s->qscale < 3 && s->max_qcoeff<=128 && s->pict_type==AV_PICTURE_TYPE_I && !(s->flags & CODEC_FLAG_QSCALE))
3532         s->qscale= 3; //reduce clipping problems
3533
3534     if (s->out_format == FMT_MJPEG) {
3535         const uint16_t *  luma_matrix = ff_mpeg1_default_intra_matrix;
3536         const uint16_t *chroma_matrix = ff_mpeg1_default_intra_matrix;
3537
3538         if (s->avctx->intra_matrix) {
3539             chroma_matrix =
3540             luma_matrix = s->avctx->intra_matrix;
3541         }
3542         if (s->avctx->chroma_intra_matrix)
3543             chroma_matrix = s->avctx->chroma_intra_matrix;
3544
3545         /* for mjpeg, we do include qscale in the matrix */
3546         for(i=1;i<64;i++){
3547             int j= s->dsp.idct_permutation[i];
3548
3549             s->chroma_intra_matrix[j] = av_clip_uint8((chroma_matrix[i] * s->qscale) >> 3);
3550             s->       intra_matrix[j] = av_clip_uint8((  luma_matrix[i] * s->qscale) >> 3);
3551         }
3552         s->y_dc_scale_table=
3553         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3554         s->chroma_intra_matrix[0] =
3555         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3556         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
3557                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3558         ff_convert_matrix(&s->dsp, s->q_chroma_intra_matrix, s->q_chroma_intra_matrix16,
3559                        s->chroma_intra_matrix, s->intra_quant_bias, 8, 8, 1);
3560         s->qscale= 8;
3561     }
3562     if(s->codec_id == AV_CODEC_ID_AMV){
3563         static const uint8_t y[32]={13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13};
3564         static const uint8_t c[32]={14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14};
3565         for(i=1;i<64;i++){
3566             int j= s->dsp.idct_permutation[ff_zigzag_direct[i]];
3567
3568             s->intra_matrix[j] = sp5x_quant_table[5*2+0][i];
3569             s->chroma_intra_matrix[j] = sp5x_quant_table[5*2+1][i];
3570         }
3571         s->y_dc_scale_table= y;
3572         s->c_dc_scale_table= c;
3573         s->intra_matrix[0] = 13;
3574         s->chroma_intra_matrix[0] = 14;
3575         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
3576                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3577         ff_convert_matrix(&s->dsp, s->q_chroma_intra_matrix, s->q_chroma_intra_matrix16,
3578                        s->chroma_intra_matrix, s->intra_quant_bias, 8, 8, 1);
3579         s->qscale= 8;
3580     }
3581
3582     //FIXME var duplication
3583     s->current_picture_ptr->f->key_frame =
3584     s->current_picture.f->key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3585     s->current_picture_ptr->f->pict_type =
3586     s->current_picture.f->pict_type = s->pict_type;
3587
3588     if (s->current_picture.f->key_frame)
3589         s->picture_in_gop_number=0;
3590
3591     s->mb_x = s->mb_y = 0;
3592     s->last_bits= put_bits_count(&s->pb);
3593     switch(s->out_format) {
3594     case FMT_MJPEG:
3595         if (CONFIG_MJPEG_ENCODER)
3596             ff_mjpeg_encode_picture_header(s->avctx, &s->pb, &s->intra_scantable,
3597                                            s->intra_matrix, s->chroma_intra_matrix);
3598         break;
3599     case FMT_H261:
3600         if (CONFIG_H261_ENCODER)
3601             ff_h261_encode_picture_header(s, picture_number);
3602         break;
3603     case FMT_H263:
3604         if (CONFIG_WMV2_ENCODER && s->codec_id == AV_CODEC_ID_WMV2)
3605             ff_wmv2_encode_picture_header(s, picture_number);
3606         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3607             ff_msmpeg4_encode_picture_header(s, picture_number);
3608         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
3609             ff_mpeg4_encode_picture_header(s, picture_number);
3610         else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10)
3611             ff_rv10_encode_picture_header(s, picture_number);
3612         else if (CONFIG_RV20_ENCODER && s->codec_id == AV_CODEC_ID_RV20)
3613             ff_rv20_encode_picture_header(s, picture_number);
3614         else if (CONFIG_FLV_ENCODER && s->codec_id == AV_CODEC_ID_FLV1)
3615             ff_flv_encode_picture_header(s, picture_number);
3616         else if (CONFIG_H263_ENCODER)
3617             ff_h263_encode_picture_header(s, picture_number);
3618         break;
3619     case FMT_MPEG1:
3620         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
3621             ff_mpeg1_encode_picture_header(s, picture_number);
3622         break;
3623     default:
3624         av_assert0(0);
3625     }
3626     bits= put_bits_count(&s->pb);
3627     s->header_bits= bits - s->last_bits;
3628
3629     for(i=1; i<context_count; i++){
3630         update_duplicate_context_after_me(s->thread_context[i], s);
3631     }
3632     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3633     for(i=1; i<context_count; i++){
3634         merge_context_after_encode(s, s->thread_context[i]);
3635     }
3636     emms_c();
3637     return 0;
3638 }
3639
3640 static void denoise_dct_c(MpegEncContext *s, int16_t *block){
3641     const int intra= s->mb_intra;
3642     int i;
3643
3644     s->dct_count[intra]++;
3645
3646     for(i=0; i<64; i++){
3647         int level= block[i];
3648
3649         if(level){
3650             if(level>0){
3651                 s->dct_error_sum[intra][i] += level;
3652                 level -= s->dct_offset[intra][i];
3653                 if(level<0) level=0;
3654             }else{
3655                 s->dct_error_sum[intra][i] -= level;
3656                 level += s->dct_offset[intra][i];
3657                 if(level>0) level=0;
3658             }
3659             block[i]= level;
3660         }
3661     }
3662 }
3663
3664 static int dct_quantize_trellis_c(MpegEncContext *s,
3665                                   int16_t *block, int n,
3666                                   int qscale, int *overflow){
3667     const int *qmat;
3668     const uint8_t *scantable= s->intra_scantable.scantable;
3669     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3670     int max=0;
3671     unsigned int threshold1, threshold2;
3672     int bias=0;
3673     int run_tab[65];
3674     int level_tab[65];
3675     int score_tab[65];
3676     int survivor[65];
3677     int survivor_count;
3678     int last_run=0;
3679     int last_level=0;
3680     int last_score= 0;
3681     int last_i;
3682     int coeff[2][64];
3683     int coeff_count[64];
3684     int qmul, qadd, start_i, last_non_zero, i, dc;
3685     const int esc_length= s->ac_esc_length;
3686     uint8_t * length;
3687     uint8_t * last_length;
3688     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3689
3690     s->dsp.fdct (block);
3691
3692     if(s->dct_error_sum)
3693         s->denoise_dct(s, block);
3694     qmul= qscale*16;
3695     qadd= ((qscale-1)|1)*8;
3696
3697     if (s->mb_intra) {
3698         int q;
3699         if (!s->h263_aic) {
3700             if (n < 4)
3701                 q = s->y_dc_scale;
3702             else
3703                 q = s->c_dc_scale;
3704             q = q << 3;
3705         } else{
3706             /* For AIC we skip quant/dequant of INTRADC */
3707             q = 1 << 3;
3708             qadd=0;
3709         }
3710
3711         /* note: block[0] is assumed to be positive */
3712         block[0] = (block[0] + (q >> 1)) / q;
3713         start_i = 1;
3714         last_non_zero = 0;
3715         qmat = n < 4 ? s->q_intra_matrix[qscale] : s->q_chroma_intra_matrix[qscale];
3716         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3717             bias= 1<<(QMAT_SHIFT-1);
3718         length     = s->intra_ac_vlc_length;
3719         last_length= s->intra_ac_vlc_last_length;
3720     } else {
3721         start_i = 0;
3722         last_non_zero = -1;
3723         qmat = s->q_inter_matrix[qscale];
3724         length     = s->inter_ac_vlc_length;
3725         last_length= s->inter_ac_vlc_last_length;
3726     }
3727     last_i= start_i;
3728
3729     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3730     threshold2= (threshold1<<1);
3731
3732     for(i=63; i>=start_i; i--) {
3733         const int j = scantable[i];
3734         int level = block[j] * qmat[j];
3735
3736         if(((unsigned)(level+threshold1))>threshold2){
3737             last_non_zero = i;
3738             break;
3739         }
3740     }
3741
3742     for(i=start_i; i<=last_non_zero; i++) {
3743         const int j = scantable[i];
3744         int level = block[j] * qmat[j];
3745
3746 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3747 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3748         if(((unsigned)(level+threshold1))>threshold2){
3749             if(level>0){
3750                 level= (bias + level)>>QMAT_SHIFT;
3751                 coeff[0][i]= level;
3752                 coeff[1][i]= level-1;
3753 //                coeff[2][k]= level-2;
3754             }else{
3755                 level= (bias - level)>>QMAT_SHIFT;
3756                 coeff[0][i]= -level;
3757                 coeff[1][i]= -level+1;
3758 //                coeff[2][k]= -level+2;
3759             }
3760             coeff_count[i]= FFMIN(level, 2);
3761             av_assert2(coeff_count[i]);
3762             max |=level;
3763         }else{
3764             coeff[0][i]= (level>>31)|1;
3765             coeff_count[i]= 1;
3766         }
3767     }
3768
3769     *overflow= s->max_qcoeff < max; //overflow might have happened
3770
3771     if(last_non_zero < start_i){
3772         memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
3773         return last_non_zero;
3774     }
3775
3776     score_tab[start_i]= 0;
3777     survivor[0]= start_i;
3778     survivor_count= 1;
3779
3780     for(i=start_i; i<=last_non_zero; i++){
3781         int level_index, j, zero_distortion;
3782         int dct_coeff= FFABS(block[ scantable[i] ]);
3783         int best_score=256*256*256*120;
3784
3785         if (s->dsp.fdct == ff_fdct_ifast)
3786             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3787         zero_distortion= dct_coeff*dct_coeff;
3788
3789         for(level_index=0; level_index < coeff_count[i]; level_index++){
3790             int distortion;
3791             int level= coeff[level_index][i];
3792             const int alevel= FFABS(level);
3793             int unquant_coeff;
3794
3795             av_assert2(level);
3796
3797             if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
3798                 unquant_coeff= alevel*qmul + qadd;
3799             }else{ //MPEG1
3800                 j= s->dsp.idct_permutation[ scantable[i] ]; //FIXME optimize
3801                 if(s->mb_intra){
3802                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3803                         unquant_coeff =   (unquant_coeff - 1) | 1;
3804                 }else{
3805                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3806                         unquant_coeff =   (unquant_coeff - 1) | 1;
3807                 }
3808                 unquant_coeff<<= 3;
3809             }
3810
3811             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3812             level+=64;
3813             if((level&(~127)) == 0){
3814                 for(j=survivor_count-1; j>=0; j--){
3815                     int run= i - survivor[j];
3816                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3817                     score += score_tab[i-run];
3818
3819                     if(score < best_score){
3820                         best_score= score;
3821                         run_tab[i+1]= run;
3822                         level_tab[i+1]= level-64;
3823                     }
3824                 }
3825
3826                 if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
3827                     for(j=survivor_count-1; j>=0; j--){
3828                         int run= i - survivor[j];
3829                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3830                         score += score_tab[i-run];
3831                         if(score < last_score){
3832                             last_score= score;
3833                             last_run= run;
3834                             last_level= level-64;
3835                             last_i= i+1;
3836                         }
3837                     }
3838                 }
3839             }else{
3840                 distortion += esc_length*lambda;
3841                 for(j=survivor_count-1; j>=0; j--){
3842                     int run= i - survivor[j];
3843                     int score= distortion + score_tab[i-run];
3844
3845                     if(score < best_score){
3846                         best_score= score;
3847                         run_tab[i+1]= run;
3848                         level_tab[i+1]= level-64;
3849                     }
3850                 }
3851
3852                 if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
3853                   for(j=survivor_count-1; j>=0; j--){
3