Merge remote-tracking branch 'qatar/master'
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of FFmpeg.
9  *
10  * FFmpeg is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * FFmpeg is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with FFmpeg; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include "libavutil/intmath.h"
31 #include "libavutil/mathematics.h"
32 #include "libavutil/opt.h"
33 #include "avcodec.h"
34 #include "dsputil.h"
35 #include "mpegvideo.h"
36 #include "mpegvideo_common.h"
37 #include "h263.h"
38 #include "mjpegenc.h"
39 #include "msmpeg4.h"
40 #include "faandct.h"
41 #include "thread.h"
42 #include "aandcttab.h"
43 #include "flv.h"
44 #include "mpeg4video.h"
45 #include "internal.h"
46 #include <limits.h>
47 #include "sp5x.h"
48
49 //#undef NDEBUG
50 //#include <assert.h>
51
52 static int encode_picture(MpegEncContext *s, int picture_number);
53 static int dct_quantize_refine(MpegEncContext *s, DCTELEM *block, int16_t *weight, DCTELEM *orig, int n, int qscale);
54 static int sse_mb(MpegEncContext *s);
55 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block);
56 static int dct_quantize_trellis_c(MpegEncContext *s, DCTELEM *block, int n, int qscale, int *overflow);
57
58 /* enable all paranoid tests for rounding, overflows, etc... */
59 //#define PARANOID
60
61 //#define DEBUG
62
63 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
64 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
65
66 void ff_convert_matrix(DSPContext *dsp, int (*qmat)[64],
67                        uint16_t (*qmat16)[2][64],
68                        const uint16_t *quant_matrix,
69                        int bias, int qmin, int qmax, int intra)
70 {
71     int qscale;
72     int shift = 0;
73
74     for (qscale = qmin; qscale <= qmax; qscale++) {
75         int i;
76         if (dsp->fdct == ff_jpeg_fdct_islow_8 ||
77             dsp->fdct == ff_jpeg_fdct_islow_10
78 #ifdef FAAN_POSTSCALE
79             || dsp->fdct == ff_faandct
80 #endif
81             ) {
82             for (i = 0; i < 64; i++) {
83                 const int j = dsp->idct_permutation[i];
84                 /* 16 <= qscale * quant_matrix[i] <= 7905
85                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
86                  *             19952 <=              x  <= 249205026
87                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
88                  *           3444240 >= (1 << 36) / (x) >= 275 */
89
90                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
91                                         (qscale * quant_matrix[j]));
92             }
93         } else if (dsp->fdct == fdct_ifast
94 #ifndef FAAN_POSTSCALE
95                    || dsp->fdct == ff_faandct
96 #endif
97                    ) {
98             for (i = 0; i < 64; i++) {
99                 const int j = dsp->idct_permutation[i];
100                 /* 16 <= qscale * quant_matrix[i] <= 7905
101                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
102                  *             19952 <=              x  <= 249205026
103                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
104                  *           3444240 >= (1 << 36) / (x) >= 275 */
105
106                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) /
107                                         (ff_aanscales[i] * qscale * quant_matrix[j]));
108             }
109         } else {
110             for (i = 0; i < 64; i++) {
111                 const int j = dsp->idct_permutation[i];
112                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
113                  * Assume x = qscale * quant_matrix[i]
114                  * So             16 <=              x  <= 7905
115                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
116                  * so          32768 >= (1 << 19) / (x) >= 67 */
117                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
118                                         (qscale * quant_matrix[j]));
119                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
120                 //                    (qscale * quant_matrix[i]);
121                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) /
122                                        (qscale * quant_matrix[j]);
123
124                 if (qmat16[qscale][0][i] == 0 ||
125                     qmat16[qscale][0][i] == 128 * 256)
126                     qmat16[qscale][0][i] = 128 * 256 - 1;
127                 qmat16[qscale][1][i] =
128                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
129                                 qmat16[qscale][0][i]);
130             }
131         }
132
133         for (i = intra; i < 64; i++) {
134             int64_t max = 8191;
135             if (dsp->fdct == fdct_ifast
136 #ifndef FAAN_POSTSCALE
137                 || dsp->fdct == ff_faandct
138 #endif
139                ) {
140                 max = (8191LL * ff_aanscales[i]) >> 14;
141             }
142             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
143                 shift++;
144             }
145         }
146     }
147     if (shift) {
148         av_log(NULL, AV_LOG_INFO,
149                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
150                QMAT_SHIFT - shift);
151     }
152 }
153
154 static inline void update_qscale(MpegEncContext *s)
155 {
156     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
157                 (FF_LAMBDA_SHIFT + 7);
158     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
159
160     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
161                  FF_LAMBDA_SHIFT;
162 }
163
164 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
165 {
166     int i;
167
168     if (matrix) {
169         put_bits(pb, 1, 1);
170         for (i = 0; i < 64; i++) {
171             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
172         }
173     } else
174         put_bits(pb, 1, 0);
175 }
176
177 /**
178  * init s->current_picture.qscale_table from s->lambda_table
179  */
180 void ff_init_qscale_tab(MpegEncContext *s)
181 {
182     int8_t * const qscale_table = s->current_picture.f.qscale_table;
183     int i;
184
185     for (i = 0; i < s->mb_num; i++) {
186         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
187         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
188         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
189                                                   s->avctx->qmax);
190     }
191 }
192
193 static void copy_picture_attributes(MpegEncContext *s,
194                                     AVFrame *dst,
195                                     AVFrame *src)
196 {
197     int i;
198
199     dst->pict_type              = src->pict_type;
200     dst->quality                = src->quality;
201     dst->coded_picture_number   = src->coded_picture_number;
202     dst->display_picture_number = src->display_picture_number;
203     //dst->reference              = src->reference;
204     dst->pts                    = src->pts;
205     dst->interlaced_frame       = src->interlaced_frame;
206     dst->top_field_first        = src->top_field_first;
207
208     if (s->avctx->me_threshold) {
209         if (!src->motion_val[0])
210             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_val not set!\n");
211         if (!src->mb_type)
212             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.mb_type not set!\n");
213         if (!src->ref_index[0])
214             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.ref_index not set!\n");
215         if (src->motion_subsample_log2 != dst->motion_subsample_log2)
216             av_log(s->avctx, AV_LOG_ERROR,
217                    "AVFrame.motion_subsample_log2 doesn't match! (%d!=%d)\n",
218                    src->motion_subsample_log2, dst->motion_subsample_log2);
219
220         memcpy(dst->mb_type, src->mb_type,
221                s->mb_stride * s->mb_height * sizeof(dst->mb_type[0]));
222
223         for (i = 0; i < 2; i++) {
224             int stride = ((16 * s->mb_width ) >>
225                           src->motion_subsample_log2) + 1;
226             int height = ((16 * s->mb_height) >> src->motion_subsample_log2);
227
228             if (src->motion_val[i] &&
229                 src->motion_val[i] != dst->motion_val[i]) {
230                 memcpy(dst->motion_val[i], src->motion_val[i],
231                        2 * stride * height * sizeof(int16_t));
232             }
233             if (src->ref_index[i] && src->ref_index[i] != dst->ref_index[i]) {
234                 memcpy(dst->ref_index[i], src->ref_index[i],
235                        s->mb_stride * 4 * s->mb_height * sizeof(int8_t));
236             }
237         }
238     }
239 }
240
241 static void update_duplicate_context_after_me(MpegEncContext *dst,
242                                               MpegEncContext *src)
243 {
244 #define COPY(a) dst->a= src->a
245     COPY(pict_type);
246     COPY(current_picture);
247     COPY(f_code);
248     COPY(b_code);
249     COPY(qscale);
250     COPY(lambda);
251     COPY(lambda2);
252     COPY(picture_in_gop_number);
253     COPY(gop_picture_number);
254     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
255     COPY(progressive_frame);    // FIXME don't set in encode_header
256     COPY(partitioned_frame);    // FIXME don't set in encode_header
257 #undef COPY
258 }
259
260 /**
261  * Set the given MpegEncContext to defaults for encoding.
262  * the changed fields will not depend upon the prior state of the MpegEncContext.
263  */
264 static void MPV_encode_defaults(MpegEncContext *s)
265 {
266     int i;
267     MPV_common_defaults(s);
268
269     for (i = -16; i < 16; i++) {
270         default_fcode_tab[i + MAX_MV] = 1;
271     }
272     s->me.mv_penalty = default_mv_penalty;
273     s->fcode_tab     = default_fcode_tab;
274 }
275
276 /* init video encoder */
277 av_cold int MPV_encode_init(AVCodecContext *avctx)
278 {
279     MpegEncContext *s = avctx->priv_data;
280     int i;
281     int chroma_h_shift, chroma_v_shift;
282
283     MPV_encode_defaults(s);
284
285     switch (avctx->codec_id) {
286     case CODEC_ID_MPEG2VIDEO:
287         if (avctx->pix_fmt != PIX_FMT_YUV420P &&
288             avctx->pix_fmt != PIX_FMT_YUV422P) {
289             av_log(avctx, AV_LOG_ERROR,
290                    "only YUV420 and YUV422 are supported\n");
291             return -1;
292         }
293         break;
294     case CODEC_ID_LJPEG:
295         if (avctx->pix_fmt != PIX_FMT_YUVJ420P &&
296             avctx->pix_fmt != PIX_FMT_YUVJ422P &&
297             avctx->pix_fmt != PIX_FMT_YUVJ444P &&
298             avctx->pix_fmt != PIX_FMT_BGRA     &&
299             ((avctx->pix_fmt != PIX_FMT_YUV420P &&
300               avctx->pix_fmt != PIX_FMT_YUV422P &&
301               avctx->pix_fmt != PIX_FMT_YUV444P) ||
302              avctx->strict_std_compliance > FF_COMPLIANCE_UNOFFICIAL)) {
303             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in LJPEG\n");
304             return -1;
305         }
306         break;
307     case CODEC_ID_MJPEG:
308     case CODEC_ID_AMV:
309         if (avctx->pix_fmt != PIX_FMT_YUVJ420P &&
310             avctx->pix_fmt != PIX_FMT_YUVJ422P &&
311             ((avctx->pix_fmt != PIX_FMT_YUV420P &&
312               avctx->pix_fmt != PIX_FMT_YUV422P) ||
313              avctx->strict_std_compliance > FF_COMPLIANCE_UNOFFICIAL)) {
314             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
315             return -1;
316         }
317         break;
318     default:
319         if (avctx->pix_fmt != PIX_FMT_YUV420P) {
320             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
321             return -1;
322         }
323     }
324
325     switch (avctx->pix_fmt) {
326     case PIX_FMT_YUVJ422P:
327     case PIX_FMT_YUV422P:
328         s->chroma_format = CHROMA_422;
329         break;
330     case PIX_FMT_YUVJ420P:
331     case PIX_FMT_YUV420P:
332     default:
333         s->chroma_format = CHROMA_420;
334         break;
335     }
336
337     s->bit_rate = avctx->bit_rate;
338     s->width    = avctx->width;
339     s->height   = avctx->height;
340     if (avctx->gop_size > 600 &&
341         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
342         av_log(avctx, AV_LOG_ERROR,
343                "Warning keyframe interval too large! reducing it ...\n");
344         avctx->gop_size = 600;
345     }
346     s->gop_size     = avctx->gop_size;
347     s->avctx        = avctx;
348     s->flags        = avctx->flags;
349     s->flags2       = avctx->flags2;
350     s->max_b_frames = avctx->max_b_frames;
351     s->codec_id     = avctx->codec->id;
352     s->luma_elim_threshold   = avctx->luma_elim_threshold;
353     s->chroma_elim_threshold = avctx->chroma_elim_threshold;
354     s->strict_std_compliance = avctx->strict_std_compliance;
355 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
356     if (avctx->flags & CODEC_FLAG_PART)
357         s->data_partitioning = 1;
358 #endif
359     s->quarter_sample     = (avctx->flags & CODEC_FLAG_QPEL) != 0;
360     s->mpeg_quant         = avctx->mpeg_quant;
361     s->rtp_mode           = !!avctx->rtp_payload_size;
362     s->intra_dc_precision = avctx->intra_dc_precision;
363     s->user_specified_pts = AV_NOPTS_VALUE;
364
365     if (s->gop_size <= 1) {
366         s->intra_only = 1;
367         s->gop_size   = 12;
368     } else {
369         s->intra_only = 0;
370     }
371
372     s->me_method = avctx->me_method;
373
374     /* Fixed QSCALE */
375     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
376
377     s->adaptive_quant = (s->avctx->lumi_masking ||
378                          s->avctx->dark_masking ||
379                          s->avctx->temporal_cplx_masking ||
380                          s->avctx->spatial_cplx_masking  ||
381                          s->avctx->p_masking      ||
382                          s->avctx->border_masking ||
383                          (s->flags & CODEC_FLAG_QP_RD)) &&
384                         !s->fixed_qscale;
385
386     s->loop_filter      = !!(s->flags & CODEC_FLAG_LOOP_FILTER);
387 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
388     s->alternate_scan   = !!(s->flags  & CODEC_FLAG_ALT_SCAN);
389     s->intra_vlc_format = !!(s->flags2 & CODEC_FLAG2_INTRA_VLC);
390     s->q_scale_type     = !!(s->flags2 & CODEC_FLAG2_NON_LINEAR_QUANT);
391     s->obmc             = !!(s->flags  & CODEC_FLAG_OBMC);
392 #endif
393
394     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
395         av_log(avctx, AV_LOG_ERROR,
396                "a vbv buffer size is needed, "
397                "for encoding with a maximum bitrate\n");
398         return -1;
399     }
400
401     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
402         av_log(avctx, AV_LOG_INFO,
403                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
404     }
405
406     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
407         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
408         return -1;
409     }
410
411     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
412         av_log(avctx, AV_LOG_ERROR, "bitrate above max bitrate\n");
413         return -1;
414     }
415
416     if (avctx->rc_max_rate &&
417         avctx->rc_max_rate == avctx->bit_rate &&
418         avctx->rc_max_rate != avctx->rc_min_rate) {
419         av_log(avctx, AV_LOG_INFO,
420                "impossible bitrate constraints, this will fail\n");
421     }
422
423     if (avctx->rc_buffer_size &&
424         avctx->bit_rate * (int64_t)avctx->time_base.num >
425             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
426         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
427         return -1;
428     }
429
430     if (!s->fixed_qscale &&
431         avctx->bit_rate * av_q2d(avctx->time_base) >
432             avctx->bit_rate_tolerance) {
433         av_log(avctx, AV_LOG_ERROR,
434                "bitrate tolerance too small for bitrate\n");
435         return -1;
436     }
437
438     if (s->avctx->rc_max_rate &&
439         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
440         (s->codec_id == CODEC_ID_MPEG1VIDEO ||
441          s->codec_id == CODEC_ID_MPEG2VIDEO) &&
442         90000LL * (avctx->rc_buffer_size - 1) >
443             s->avctx->rc_max_rate * 0xFFFFLL) {
444         av_log(avctx, AV_LOG_INFO,
445                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
446                "specified vbv buffer is too large for the given bitrate!\n");
447     }
448
449     if ((s->flags & CODEC_FLAG_4MV)  && s->codec_id != CODEC_ID_MPEG4 &&
450         s->codec_id != CODEC_ID_H263 && s->codec_id != CODEC_ID_H263P &&
451         s->codec_id != CODEC_ID_FLV1) {
452         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
453         return -1;
454     }
455
456     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
457         av_log(avctx, AV_LOG_ERROR,
458                "OBMC is only supported with simple mb decision\n");
459         return -1;
460     }
461
462 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
463     if (s->obmc && s->codec_id != CODEC_ID_H263 &&
464         s->codec_id != CODEC_ID_H263P) {
465         av_log(avctx, AV_LOG_ERROR, "OBMC is only supported with H263(+)\n");
466         return -1;
467     }
468 #endif
469
470     if (s->quarter_sample && s->codec_id != CODEC_ID_MPEG4) {
471         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
472         return -1;
473     }
474
475 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
476     if (s->data_partitioning && s->codec_id != CODEC_ID_MPEG4) {
477         av_log(avctx, AV_LOG_ERROR,
478                "data partitioning not supported by codec\n");
479         return -1;
480     }
481 #endif
482
483     if (s->max_b_frames                    &&
484         s->codec_id != CODEC_ID_MPEG4      &&
485         s->codec_id != CODEC_ID_MPEG1VIDEO &&
486         s->codec_id != CODEC_ID_MPEG2VIDEO) {
487         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
488         return -1;
489     }
490
491     if ((s->codec_id == CODEC_ID_MPEG4 ||
492          s->codec_id == CODEC_ID_H263  ||
493          s->codec_id == CODEC_ID_H263P) &&
494         (avctx->sample_aspect_ratio.num > 255 ||
495          avctx->sample_aspect_ratio.den > 255)) {
496         av_log(avctx, AV_LOG_WARNING,
497                "Invalid pixel aspect ratio %i/%i, limit is 255/255 reducing\n",
498                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
499         av_reduce(&avctx->sample_aspect_ratio.num, &avctx->sample_aspect_ratio.den,
500                    avctx->sample_aspect_ratio.num,  avctx->sample_aspect_ratio.den, 255);
501     }
502
503     if ((s->flags & (CODEC_FLAG_INTERLACED_DCT | CODEC_FLAG_INTERLACED_ME |
504                      CODEC_FLAG_ALT_SCAN)) &&
505         s->codec_id != CODEC_ID_MPEG4 && s->codec_id != CODEC_ID_MPEG2VIDEO) {
506         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
507         return -1;
508     }
509
510     // FIXME mpeg2 uses that too
511     if (s->mpeg_quant && s->codec_id != CODEC_ID_MPEG4) {
512         av_log(avctx, AV_LOG_ERROR,
513                "mpeg2 style quantization not supported by codec\n");
514         return -1;
515     }
516
517     if ((s->flags & CODEC_FLAG_CBP_RD) && !avctx->trellis) {
518         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
519         return -1;
520     }
521
522     if ((s->flags & CODEC_FLAG_QP_RD) &&
523         s->avctx->mb_decision != FF_MB_DECISION_RD) {
524         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
525         return -1;
526     }
527
528     if (s->avctx->scenechange_threshold < 1000000000 &&
529         (s->flags & CODEC_FLAG_CLOSED_GOP)) {
530         av_log(avctx, AV_LOG_ERROR,
531                "closed gop with scene change detection are not supported yet, "
532                "set threshold to 1000000000\n");
533         return -1;
534     }
535
536     if ((s->flags2 & CODEC_FLAG2_INTRA_VLC) &&
537         s->codec_id != CODEC_ID_MPEG2VIDEO) {
538         av_log(avctx, AV_LOG_ERROR,
539                "intra vlc table not supported by codec\n");
540         return -1;
541     }
542
543     if (s->flags & CODEC_FLAG_LOW_DELAY) {
544         if (s->codec_id != CODEC_ID_MPEG2VIDEO) {
545             av_log(avctx, AV_LOG_ERROR,
546                   "low delay forcing is only available for mpeg2\n");
547             return -1;
548         }
549         if (s->max_b_frames != 0) {
550             av_log(avctx, AV_LOG_ERROR,
551                    "b frames cannot be used with low delay\n");
552             return -1;
553         }
554     }
555
556     if (s->q_scale_type == 1) {
557 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
558         if (s->codec_id != CODEC_ID_MPEG2VIDEO) {
559             av_log(avctx, AV_LOG_ERROR,
560                    "non linear quant is only available for mpeg2\n");
561             return -1;
562         }
563 #endif
564         if (avctx->qmax > 12) {
565             av_log(avctx, AV_LOG_ERROR,
566                    "non linear quant only supports qmax <= 12 currently\n");
567             return -1;
568         }
569     }
570
571     if (s->avctx->thread_count > 1         &&
572         s->codec_id != CODEC_ID_MPEG4      &&
573         s->codec_id != CODEC_ID_MPEG1VIDEO &&
574         s->codec_id != CODEC_ID_MPEG2VIDEO &&
575         (s->codec_id != CODEC_ID_H263P ||
576          !(s->flags & CODEC_FLAG_H263P_SLICE_STRUCT))) {
577         av_log(avctx, AV_LOG_ERROR,
578                "multi threaded encoding not supported by codec\n");
579         return -1;
580     }
581
582     if (s->avctx->thread_count < 1) {
583         av_log(avctx, AV_LOG_ERROR,
584                "automatic thread number detection not supported by codec, "
585                "patch welcome\n");
586         return -1;
587     }
588
589     if (s->avctx->thread_count > 1)
590         s->rtp_mode = 1;
591
592     if (!avctx->time_base.den || !avctx->time_base.num) {
593         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
594         return -1;
595     }
596
597     i = (INT_MAX / 2 + 128) >> 8;
598     if (avctx->me_threshold >= i) {
599         av_log(avctx, AV_LOG_ERROR, "me_threshold too large, max is %d\n",
600                i - 1);
601         return -1;
602     }
603     if (avctx->mb_threshold >= i) {
604         av_log(avctx, AV_LOG_ERROR, "mb_threshold too large, max is %d\n",
605                i - 1);
606         return -1;
607     }
608
609     if (avctx->b_frame_strategy && (avctx->flags & CODEC_FLAG_PASS2)) {
610         av_log(avctx, AV_LOG_INFO,
611                "notice: b_frame_strategy only affects the first pass\n");
612         avctx->b_frame_strategy = 0;
613     }
614
615     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
616     if (i > 1) {
617         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
618         avctx->time_base.den /= i;
619         avctx->time_base.num /= i;
620         //return -1;
621     }
622
623     if (s->mpeg_quant || s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO || s->codec_id == CODEC_ID_MJPEG || s->codec_id==CODEC_ID_AMV) {
624         // (a + x * 3 / 8) / x
625         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
626         s->inter_quant_bias = 0;
627     } else {
628         s->intra_quant_bias = 0;
629         // (a - x / 4) / x
630         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
631     }
632
633     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
634         s->intra_quant_bias = avctx->intra_quant_bias;
635     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
636         s->inter_quant_bias = avctx->inter_quant_bias;
637
638     av_log(avctx, AV_LOG_DEBUG, "intra_quant_bias = %d inter_quant_bias = %d\n",s->intra_quant_bias,s->inter_quant_bias);
639
640     avcodec_get_chroma_sub_sample(avctx->pix_fmt, &chroma_h_shift,
641                                   &chroma_v_shift);
642
643     if (avctx->codec_id == CODEC_ID_MPEG4 &&
644         s->avctx->time_base.den > (1 << 16) - 1) {
645         av_log(avctx, AV_LOG_ERROR,
646                "timebase %d/%d not supported by MPEG 4 standard, "
647                "the maximum admitted value for the timebase denominator "
648                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
649                (1 << 16) - 1);
650         return -1;
651     }
652     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
653
654     switch (avctx->codec->id) {
655     case CODEC_ID_MPEG1VIDEO:
656         s->out_format = FMT_MPEG1;
657         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
658         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
659         break;
660     case CODEC_ID_MPEG2VIDEO:
661         s->out_format = FMT_MPEG1;
662         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
663         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
664         s->rtp_mode   = 1;
665         break;
666     case CODEC_ID_LJPEG:
667     case CODEC_ID_MJPEG:
668     case CODEC_ID_AMV:
669         s->out_format = FMT_MJPEG;
670         s->intra_only = 1; /* force intra only for jpeg */
671         if (avctx->codec->id == CODEC_ID_LJPEG && avctx->pix_fmt   == PIX_FMT_BGRA) {
672             s->mjpeg_vsample[0] = s->mjpeg_hsample[0] =
673             s->mjpeg_vsample[1] = s->mjpeg_hsample[1] =
674             s->mjpeg_vsample[2] = s->mjpeg_hsample[2] = 1;
675         } else {
676             s->mjpeg_vsample[0] = 2;
677             s->mjpeg_vsample[1] = 2 >> chroma_v_shift;
678             s->mjpeg_vsample[2] = 2 >> chroma_v_shift;
679             s->mjpeg_hsample[0] = 2;
680             s->mjpeg_hsample[1] = 2 >> chroma_h_shift;
681             s->mjpeg_hsample[2] = 2 >> chroma_h_shift;
682         }
683         if (!(CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER) ||
684             ff_mjpeg_encode_init(s) < 0)
685             return -1;
686         avctx->delay = 0;
687         s->low_delay = 1;
688         break;
689     case CODEC_ID_H261:
690         if (!CONFIG_H261_ENCODER)
691             return -1;
692         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
693             av_log(avctx, AV_LOG_ERROR,
694                    "The specified picture size of %dx%d is not valid for the "
695                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
696                     s->width, s->height);
697             return -1;
698         }
699         s->out_format = FMT_H261;
700         avctx->delay  = 0;
701         s->low_delay  = 1;
702         break;
703     case CODEC_ID_H263:
704         if (!CONFIG_H263_ENCODER)
705             return -1;
706         if (ff_match_2uint16(h263_format, FF_ARRAY_ELEMS(h263_format),
707                              s->width, s->height) == 8) {
708             av_log(avctx, AV_LOG_ERROR,
709                    "The specified picture size of %dx%d is not valid for "
710                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
711                    "352x288, 704x576, and 1408x1152. "
712                    "Try H.263+.\n", s->width, s->height);
713             return -1;
714         }
715         s->out_format = FMT_H263;
716         avctx->delay  = 0;
717         s->low_delay  = 1;
718         break;
719     case CODEC_ID_H263P:
720         s->out_format = FMT_H263;
721         s->h263_plus  = 1;
722         /* Fx */
723 #if FF_API_MPEGVIDEO_GLOBAL_OPTS
724         if (avctx->flags & CODEC_FLAG_H263P_UMV)
725             s->umvplus = 1;
726         if (avctx->flags & CODEC_FLAG_H263P_AIV)
727             s->alt_inter_vlc = 1;
728         if (avctx->flags & CODEC_FLAG_H263P_SLICE_STRUCT)
729             s->h263_slice_structured = 1;
730 #endif
731         s->h263_aic        = (avctx->flags & CODEC_FLAG_AC_PRED) ? 1 : 0;
732         s->modified_quant  = s->h263_aic;
733         s->loop_filter     = (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
734         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
735
736         /* /Fx */
737         /* These are just to be sure */
738         avctx->delay = 0;
739         s->low_delay = 1;
740         break;
741     case CODEC_ID_FLV1:
742         s->out_format      = FMT_H263;
743         s->h263_flv        = 2; /* format = 1; 11-bit codes */
744         s->unrestricted_mv = 1;
745         s->rtp_mode  = 0; /* don't allow GOB */
746         avctx->delay = 0;
747         s->low_delay = 1;
748         break;
749     case CODEC_ID_RV10:
750         s->out_format = FMT_H263;
751         avctx->delay  = 0;
752         s->low_delay  = 1;
753         break;
754     case CODEC_ID_RV20:
755         s->out_format      = FMT_H263;
756         avctx->delay       = 0;
757         s->low_delay       = 1;
758         s->modified_quant  = 1;
759         s->h263_aic        = 1;
760         s->h263_plus       = 1;
761         s->loop_filter     = 1;
762         s->unrestricted_mv = 0;
763         break;
764     case CODEC_ID_MPEG4:
765         s->out_format      = FMT_H263;
766         s->h263_pred       = 1;
767         s->unrestricted_mv = 1;
768         s->low_delay       = s->max_b_frames ? 0 : 1;
769         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
770         break;
771     case CODEC_ID_MSMPEG4V2:
772         s->out_format      = FMT_H263;
773         s->h263_pred       = 1;
774         s->unrestricted_mv = 1;
775         s->msmpeg4_version = 2;
776         avctx->delay       = 0;
777         s->low_delay       = 1;
778         break;
779     case CODEC_ID_MSMPEG4V3:
780         s->out_format        = FMT_H263;
781         s->h263_pred         = 1;
782         s->unrestricted_mv   = 1;
783         s->msmpeg4_version   = 3;
784         s->flipflop_rounding = 1;
785         avctx->delay         = 0;
786         s->low_delay         = 1;
787         break;
788     case CODEC_ID_WMV1:
789         s->out_format        = FMT_H263;
790         s->h263_pred         = 1;
791         s->unrestricted_mv   = 1;
792         s->msmpeg4_version   = 4;
793         s->flipflop_rounding = 1;
794         avctx->delay         = 0;
795         s->low_delay         = 1;
796         break;
797     case CODEC_ID_WMV2:
798         s->out_format        = FMT_H263;
799         s->h263_pred         = 1;
800         s->unrestricted_mv   = 1;
801         s->msmpeg4_version   = 5;
802         s->flipflop_rounding = 1;
803         avctx->delay         = 0;
804         s->low_delay         = 1;
805         break;
806     default:
807         return -1;
808     }
809
810     avctx->has_b_frames = !s->low_delay;
811
812     s->encoding = 1;
813
814     s->progressive_frame    =
815     s->progressive_sequence = !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT |
816                                                 CODEC_FLAG_INTERLACED_ME  |
817                                                 CODEC_FLAG_ALT_SCAN));
818
819     /* init */
820     if (MPV_common_init(s) < 0)
821         return -1;
822
823     if (!s->dct_quantize)
824         s->dct_quantize = dct_quantize_c;
825     if (!s->denoise_dct)
826         s->denoise_dct  = denoise_dct_c;
827     s->fast_dct_quantize = s->dct_quantize;
828     if (avctx->trellis)
829         s->dct_quantize  = dct_quantize_trellis_c;
830
831     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
832         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
833
834     s->quant_precision = 5;
835
836     ff_set_cmp(&s->dsp, s->dsp.ildct_cmp, s->avctx->ildct_cmp);
837     ff_set_cmp(&s->dsp, s->dsp.frame_skip_cmp, s->avctx->frame_skip_cmp);
838
839     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
840         ff_h261_encode_init(s);
841     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
842         h263_encode_init(s);
843     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
844         ff_msmpeg4_encode_init(s);
845     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
846         && s->out_format == FMT_MPEG1)
847         ff_mpeg1_encode_init(s);
848
849     /* init q matrix */
850     for (i = 0; i < 64; i++) {
851         int j = s->dsp.idct_permutation[i];
852         if (CONFIG_MPEG4_ENCODER && s->codec_id == CODEC_ID_MPEG4 &&
853             s->mpeg_quant) {
854             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
855             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
856         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
857             s->intra_matrix[j] =
858             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
859         } else {
860             /* mpeg1/2 */
861             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
862             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
863         }
864         if (s->avctx->intra_matrix)
865             s->intra_matrix[j] = s->avctx->intra_matrix[i];
866         if (s->avctx->inter_matrix)
867             s->inter_matrix[j] = s->avctx->inter_matrix[i];
868     }
869
870     /* precompute matrix */
871     /* for mjpeg, we do include qscale in the matrix */
872     if (s->out_format != FMT_MJPEG) {
873         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
874                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
875                           31, 1);
876         ff_convert_matrix(&s->dsp, s->q_inter_matrix, s->q_inter_matrix16,
877                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
878                           31, 0);
879     }
880
881     if (ff_rate_control_init(s) < 0)
882         return -1;
883
884     return 0;
885 }
886
887 av_cold int MPV_encode_end(AVCodecContext *avctx)
888 {
889     MpegEncContext *s = avctx->priv_data;
890
891     ff_rate_control_uninit(s);
892
893     MPV_common_end(s);
894     if ((CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER) &&
895         s->out_format == FMT_MJPEG)
896         ff_mjpeg_encode_close(s);
897
898     av_freep(&avctx->extradata);
899
900     return 0;
901 }
902
903 static int get_sae(uint8_t *src, int ref, int stride)
904 {
905     int x,y;
906     int acc = 0;
907
908     for (y = 0; y < 16; y++) {
909         for (x = 0; x < 16; x++) {
910             acc += FFABS(src[x + y * stride] - ref);
911         }
912     }
913
914     return acc;
915 }
916
917 static int get_intra_count(MpegEncContext *s, uint8_t *src,
918                            uint8_t *ref, int stride)
919 {
920     int x, y, w, h;
921     int acc = 0;
922
923     w = s->width  & ~15;
924     h = s->height & ~15;
925
926     for (y = 0; y < h; y += 16) {
927         for (x = 0; x < w; x += 16) {
928             int offset = x + y * stride;
929             int sad  = s->dsp.sad[0](NULL, src + offset, ref + offset, stride,
930                                      16);
931             int mean = (s->dsp.pix_sum(src + offset, stride) + 128) >> 8;
932             int sae  = get_sae(src + offset, mean, stride);
933
934             acc += sae + 500 < sad;
935         }
936     }
937     return acc;
938 }
939
940
941 static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg)
942 {
943     AVFrame *pic = NULL;
944     int64_t pts;
945     int i;
946     const int encoding_delay = s->max_b_frames;
947     int direct = 1;
948
949     if (pic_arg) {
950         pts = pic_arg->pts;
951         pic_arg->display_picture_number = s->input_picture_number++;
952
953         if (pts != AV_NOPTS_VALUE) {
954             if (s->user_specified_pts != AV_NOPTS_VALUE) {
955                 int64_t time = pts;
956                 int64_t last = s->user_specified_pts;
957
958                 if (time <= last) {
959                     av_log(s->avctx, AV_LOG_ERROR,
960                            "Error, Invalid timestamp=%"PRId64", "
961                            "last=%"PRId64"\n", pts, s->user_specified_pts);
962                     return -1;
963                 }
964             }
965             s->user_specified_pts = pts;
966         } else {
967             if (s->user_specified_pts != AV_NOPTS_VALUE) {
968                 s->user_specified_pts =
969                 pts = s->user_specified_pts + 1;
970                 av_log(s->avctx, AV_LOG_INFO,
971                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
972                        pts);
973             } else {
974                 pts = pic_arg->display_picture_number;
975             }
976         }
977     }
978
979   if (pic_arg) {
980     if (encoding_delay && !(s->flags & CODEC_FLAG_INPUT_PRESERVED))
981         direct = 0;
982     if (pic_arg->linesize[0] != s->linesize)
983         direct = 0;
984     if (pic_arg->linesize[1] != s->uvlinesize)
985         direct = 0;
986     if (pic_arg->linesize[2] != s->uvlinesize)
987         direct = 0;
988
989     //av_log(AV_LOG_DEBUG, "%d %d %d %d\n",pic_arg->linesize[0],
990     //       pic_arg->linesize[1], s->linesize, s->uvlinesize);
991
992     if (direct) {
993         i = ff_find_unused_picture(s, 1);
994         if (i < 0)
995             return i;
996
997         pic = (AVFrame *) &s->picture[i];
998         pic->reference = 3;
999
1000         for (i = 0; i < 4; i++) {
1001             pic->data[i]     = pic_arg->data[i];
1002             pic->linesize[i] = pic_arg->linesize[i];
1003         }
1004         if (ff_alloc_picture(s, (Picture *) pic, 1) < 0) {
1005             return -1;
1006         }
1007     } else {
1008         i = ff_find_unused_picture(s, 0);
1009         if (i < 0)
1010             return i;
1011
1012         pic = (AVFrame *) &s->picture[i];
1013         pic->reference = 3;
1014
1015         if (ff_alloc_picture(s, (Picture *) pic, 0) < 0) {
1016             return -1;
1017         }
1018
1019         if (pic->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1020             pic->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1021             pic->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1022             // empty
1023         } else {
1024             int h_chroma_shift, v_chroma_shift;
1025             avcodec_get_chroma_sub_sample(s->avctx->pix_fmt, &h_chroma_shift,
1026                                           &v_chroma_shift);
1027
1028             for (i = 0; i < 3; i++) {
1029                 int src_stride = pic_arg->linesize[i];
1030                 int dst_stride = i ? s->uvlinesize : s->linesize;
1031                 int h_shift = i ? h_chroma_shift : 0;
1032                 int v_shift = i ? v_chroma_shift : 0;
1033                 int w = s->width  >> h_shift;
1034                 int h = s->height >> v_shift;
1035                 uint8_t *src = pic_arg->data[i];
1036                 uint8_t *dst = pic->data[i];
1037
1038                 if(s->codec_id == CODEC_ID_AMV && !(s->avctx->flags & CODEC_FLAG_EMU_EDGE)){
1039                     h= ((s->height+15)/16*16)>>v_shift;
1040                 }
1041
1042                 if (!s->avctx->rc_buffer_size)
1043                     dst += INPLACE_OFFSET;
1044
1045                 if (src_stride == dst_stride)
1046                     memcpy(dst, src, src_stride * h);
1047                 else {
1048                     while (h--) {
1049                         memcpy(dst, src, w);
1050                         dst += dst_stride;
1051                         src += src_stride;
1052                     }
1053                 }
1054             }
1055         }
1056     }
1057     copy_picture_attributes(s, pic, pic_arg);
1058     pic->pts = pts; // we set this here to avoid modifiying pic_arg
1059   }
1060
1061     /* shift buffer entries */
1062     for (i = 1; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1063         s->input_picture[i - 1] = s->input_picture[i];
1064
1065     s->input_picture[encoding_delay] = (Picture*) pic;
1066
1067     return 0;
1068 }
1069
1070 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1071 {
1072     int x, y, plane;
1073     int score = 0;
1074     int64_t score64 = 0;
1075
1076     for (plane = 0; plane < 3; plane++) {
1077         const int stride = p->f.linesize[plane];
1078         const int bw = plane ? 1 : 2;
1079         for (y = 0; y < s->mb_height * bw; y++) {
1080             for (x = 0; x < s->mb_width * bw; x++) {
1081                 int off = p->f.type == FF_BUFFER_TYPE_SHARED ? 0 : 16;
1082                 uint8_t *dptr = p->f.data[plane] + 8 * (x + y * stride) + off;
1083                 uint8_t *rptr = ref->f.data[plane] + 8 * (x + y * stride);
1084                 int v   = s->dsp.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1085
1086                 switch (s->avctx->frame_skip_exp) {
1087                 case 0: score    =  FFMAX(score, v);          break;
1088                 case 1: score   += FFABS(v);                  break;
1089                 case 2: score   += v * v;                     break;
1090                 case 3: score64 += FFABS(v * v * (int64_t)v); break;
1091                 case 4: score64 += v * v * (int64_t)(v * v);  break;
1092                 }
1093             }
1094         }
1095     }
1096
1097     if (score)
1098         score64 = score;
1099
1100     if (score64 < s->avctx->frame_skip_threshold)
1101         return 1;
1102     if (score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda) >> 8))
1103         return 1;
1104     return 0;
1105 }
1106
1107 static int estimate_best_b_count(MpegEncContext *s)
1108 {
1109     AVCodec *codec    = avcodec_find_encoder(s->avctx->codec_id);
1110     AVCodecContext *c = avcodec_alloc_context3(NULL);
1111     AVFrame input[FF_MAX_B_FRAMES + 2];
1112     const int scale = s->avctx->brd_scale;
1113     int i, j, out_size, p_lambda, b_lambda, lambda2;
1114     int outbuf_size  = s->width * s->height; // FIXME
1115     uint8_t *outbuf  = av_malloc(outbuf_size);
1116     int64_t best_rd  = INT64_MAX;
1117     int best_b_count = -1;
1118
1119     assert(scale >= 0 && scale <= 3);
1120
1121     //emms_c();
1122     //s->next_picture_ptr->quality;
1123     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1124     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1125     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1126     if (!b_lambda) // FIXME we should do this somewhere else
1127         b_lambda = p_lambda;
1128     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1129                FF_LAMBDA_SHIFT;
1130
1131     c->width        = s->width  >> scale;
1132     c->height       = s->height >> scale;
1133     c->flags        = CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR |
1134                       CODEC_FLAG_INPUT_PRESERVED /*| CODEC_FLAG_EMU_EDGE*/;
1135     c->flags       |= s->avctx->flags & CODEC_FLAG_QPEL;
1136     c->mb_decision  = s->avctx->mb_decision;
1137     c->me_cmp       = s->avctx->me_cmp;
1138     c->mb_cmp       = s->avctx->mb_cmp;
1139     c->me_sub_cmp   = s->avctx->me_sub_cmp;
1140     c->pix_fmt      = PIX_FMT_YUV420P;
1141     c->time_base    = s->avctx->time_base;
1142     c->max_b_frames = s->max_b_frames;
1143
1144     if (avcodec_open2(c, codec, NULL) < 0)
1145         return -1;
1146
1147     for (i = 0; i < s->max_b_frames + 2; i++) {
1148         int ysize = c->width * c->height;
1149         int csize = (c->width / 2) * (c->height / 2);
1150         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1151                                                 s->next_picture_ptr;
1152
1153         avcodec_get_frame_defaults(&input[i]);
1154         input[i].data[0]     = av_malloc(ysize + 2 * csize);
1155         input[i].data[1]     = input[i].data[0] + ysize;
1156         input[i].data[2]     = input[i].data[1] + csize;
1157         input[i].linesize[0] = c->width;
1158         input[i].linesize[1] =
1159         input[i].linesize[2] = c->width / 2;
1160
1161         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1162             pre_input = *pre_input_ptr;
1163
1164             if (pre_input.f.type != FF_BUFFER_TYPE_SHARED && i) {
1165                 pre_input.f.data[0] += INPLACE_OFFSET;
1166                 pre_input.f.data[1] += INPLACE_OFFSET;
1167                 pre_input.f.data[2] += INPLACE_OFFSET;
1168             }
1169
1170             s->dsp.shrink[scale](input[i].data[0], input[i].linesize[0],
1171                                  pre_input.f.data[0], pre_input.f.linesize[0],
1172                                  c->width,      c->height);
1173             s->dsp.shrink[scale](input[i].data[1], input[i].linesize[1],
1174                                  pre_input.f.data[1], pre_input.f.linesize[1],
1175                                  c->width >> 1, c->height >> 1);
1176             s->dsp.shrink[scale](input[i].data[2], input[i].linesize[2],
1177                                  pre_input.f.data[2], pre_input.f.linesize[2],
1178                                  c->width >> 1, c->height >> 1);
1179         }
1180     }
1181
1182     for (j = 0; j < s->max_b_frames + 1; j++) {
1183         int64_t rd = 0;
1184
1185         if (!s->input_picture[j])
1186             break;
1187
1188         c->error[0] = c->error[1] = c->error[2] = 0;
1189
1190         input[0].pict_type = AV_PICTURE_TYPE_I;
1191         input[0].quality   = 1 * FF_QP2LAMBDA;
1192         out_size           = avcodec_encode_video(c, outbuf,
1193                                                   outbuf_size, &input[0]);
1194         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1195
1196         for (i = 0; i < s->max_b_frames + 1; i++) {
1197             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1198
1199             input[i + 1].pict_type = is_p ?
1200                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1201             input[i + 1].quality   = is_p ? p_lambda : b_lambda;
1202             out_size = avcodec_encode_video(c, outbuf, outbuf_size,
1203                                             &input[i + 1]);
1204             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1205         }
1206
1207         /* get the delayed frames */
1208         while (out_size) {
1209             out_size = avcodec_encode_video(c, outbuf, outbuf_size, NULL);
1210             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1211         }
1212
1213         rd += c->error[0] + c->error[1] + c->error[2];
1214
1215         if (rd < best_rd) {
1216             best_rd = rd;
1217             best_b_count = j;
1218         }
1219     }
1220
1221     av_freep(&outbuf);
1222     avcodec_close(c);
1223     av_freep(&c);
1224
1225     for (i = 0; i < s->max_b_frames + 2; i++) {
1226         av_freep(&input[i].data[0]);
1227     }
1228
1229     return best_b_count;
1230 }
1231
1232 static int select_input_picture(MpegEncContext *s)
1233 {
1234     int i;
1235
1236     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1237         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1238     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1239
1240     /* set next picture type & ordering */
1241     if (s->reordered_input_picture[0] == NULL && s->input_picture[0]) {
1242         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1243             s->next_picture_ptr == NULL || s->intra_only) {
1244             s->reordered_input_picture[0] = s->input_picture[0];
1245             s->reordered_input_picture[0]->f.pict_type = AV_PICTURE_TYPE_I;
1246             s->reordered_input_picture[0]->f.coded_picture_number =
1247                 s->coded_picture_number++;
1248         } else {
1249             int b_frames;
1250
1251             if (s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor) {
1252                 if (s->picture_in_gop_number < s->gop_size &&
1253                     skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1254                     // FIXME check that te gop check above is +-1 correct
1255                     //av_log(NULL, AV_LOG_DEBUG, "skip %p %"PRId64"\n",
1256                     //       s->input_picture[0]->f.data[0],
1257                     //       s->input_picture[0]->pts);
1258
1259                     if (s->input_picture[0]->f.type == FF_BUFFER_TYPE_SHARED) {
1260                         for (i = 0; i < 4; i++)
1261                             s->input_picture[0]->f.data[i] = NULL;
1262                         s->input_picture[0]->f.type = 0;
1263                     } else {
1264                         assert(s->input_picture[0]->f.type == FF_BUFFER_TYPE_USER ||
1265                                s->input_picture[0]->f.type == FF_BUFFER_TYPE_INTERNAL);
1266
1267                         s->avctx->release_buffer(s->avctx,
1268                                                  (AVFrame *) s->input_picture[0]);
1269                     }
1270
1271                     emms_c();
1272                     ff_vbv_update(s, 0);
1273
1274                     goto no_output_pic;
1275                 }
1276             }
1277
1278             if (s->flags & CODEC_FLAG_PASS2) {
1279                 for (i = 0; i < s->max_b_frames + 1; i++) {
1280                     int pict_num = s->input_picture[0]->f.display_picture_number + i;
1281
1282                     if (pict_num >= s->rc_context.num_entries)
1283                         break;
1284                     if (!s->input_picture[i]) {
1285                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1286                         break;
1287                     }
1288
1289                     s->input_picture[i]->f.pict_type =
1290                         s->rc_context.entry[pict_num].new_pict_type;
1291                 }
1292             }
1293
1294             if (s->avctx->b_frame_strategy == 0) {
1295                 b_frames = s->max_b_frames;
1296                 while (b_frames && !s->input_picture[b_frames])
1297                     b_frames--;
1298             } else if (s->avctx->b_frame_strategy == 1) {
1299                 for (i = 1; i < s->max_b_frames + 1; i++) {
1300                     if (s->input_picture[i] &&
1301                         s->input_picture[i]->b_frame_score == 0) {
1302                         s->input_picture[i]->b_frame_score =
1303                             get_intra_count(s,
1304                                             s->input_picture[i    ]->f.data[0],
1305                                             s->input_picture[i - 1]->f.data[0],
1306                                             s->linesize) + 1;
1307                     }
1308                 }
1309                 for (i = 0; i < s->max_b_frames + 1; i++) {
1310                     if (s->input_picture[i] == NULL ||
1311                         s->input_picture[i]->b_frame_score - 1 >
1312                             s->mb_num / s->avctx->b_sensitivity)
1313                         break;
1314                 }
1315
1316                 b_frames = FFMAX(0, i - 1);
1317
1318                 /* reset scores */
1319                 for (i = 0; i < b_frames + 1; i++) {
1320                     s->input_picture[i]->b_frame_score = 0;
1321                 }
1322             } else if (s->avctx->b_frame_strategy == 2) {
1323                 b_frames = estimate_best_b_count(s);
1324             } else {
1325                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1326                 b_frames = 0;
1327             }
1328
1329             emms_c();
1330             //static int b_count = 0;
1331             //b_count += b_frames;
1332             //av_log(s->avctx, AV_LOG_DEBUG, "b_frames: %d\n", b_count);
1333
1334             for (i = b_frames - 1; i >= 0; i--) {
1335                 int type = s->input_picture[i]->f.pict_type;
1336                 if (type && type != AV_PICTURE_TYPE_B)
1337                     b_frames = i;
1338             }
1339             if (s->input_picture[b_frames]->f.pict_type == AV_PICTURE_TYPE_B &&
1340                 b_frames == s->max_b_frames) {
1341                 av_log(s->avctx, AV_LOG_ERROR,
1342                        "warning, too many b frames in a row\n");
1343             }
1344
1345             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1346                 if ((s->flags2 & CODEC_FLAG2_STRICT_GOP) &&
1347                     s->gop_size > s->picture_in_gop_number) {
1348                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1349                 } else {
1350                     if (s->flags & CODEC_FLAG_CLOSED_GOP)
1351                         b_frames = 0;
1352                     s->input_picture[b_frames]->f.pict_type = AV_PICTURE_TYPE_I;
1353                 }
1354             }
1355
1356             if ((s->flags & CODEC_FLAG_CLOSED_GOP) && b_frames &&
1357                 s->input_picture[b_frames]->f.pict_type == AV_PICTURE_TYPE_I)
1358                 b_frames--;
1359
1360             s->reordered_input_picture[0] = s->input_picture[b_frames];
1361             if (s->reordered_input_picture[0]->f.pict_type != AV_PICTURE_TYPE_I)
1362                 s->reordered_input_picture[0]->f.pict_type = AV_PICTURE_TYPE_P;
1363             s->reordered_input_picture[0]->f.coded_picture_number =
1364                 s->coded_picture_number++;
1365             for (i = 0; i < b_frames; i++) {
1366                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1367                 s->reordered_input_picture[i + 1]->f.pict_type =
1368                     AV_PICTURE_TYPE_B;
1369                 s->reordered_input_picture[i + 1]->f.coded_picture_number =
1370                     s->coded_picture_number++;
1371             }
1372         }
1373     }
1374 no_output_pic:
1375     if (s->reordered_input_picture[0]) {
1376         s->reordered_input_picture[0]->f.reference =
1377            s->reordered_input_picture[0]->f.pict_type !=
1378                AV_PICTURE_TYPE_B ? 3 : 0;
1379
1380         ff_copy_picture(&s->new_picture, s->reordered_input_picture[0]);
1381
1382         if (s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_SHARED ||
1383             s->avctx->rc_buffer_size) {
1384             // input is a shared pix, so we can't modifiy it -> alloc a new
1385             // one & ensure that the shared one is reuseable
1386
1387             Picture *pic;
1388             int i = ff_find_unused_picture(s, 0);
1389             if (i < 0)
1390                 return i;
1391             pic = &s->picture[i];
1392
1393             pic->f.reference = s->reordered_input_picture[0]->f.reference;
1394             if (ff_alloc_picture(s, pic, 0) < 0) {
1395                 return -1;
1396             }
1397
1398             /* mark us unused / free shared pic */
1399             if (s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_INTERNAL)
1400                 s->avctx->release_buffer(s->avctx,
1401                                          (AVFrame *) s->reordered_input_picture[0]);
1402             for (i = 0; i < 4; i++)
1403                 s->reordered_input_picture[0]->f.data[i] = NULL;
1404             s->reordered_input_picture[0]->f.type = 0;
1405
1406             copy_picture_attributes(s, (AVFrame *) pic,
1407                                     (AVFrame *) s->reordered_input_picture[0]);
1408
1409             s->current_picture_ptr = pic;
1410         } else {
1411             // input is not a shared pix -> reuse buffer for current_pix
1412
1413             assert(s->reordered_input_picture[0]->f.type ==
1414                        FF_BUFFER_TYPE_USER ||
1415                    s->reordered_input_picture[0]->f.type ==
1416                        FF_BUFFER_TYPE_INTERNAL);
1417
1418             s->current_picture_ptr = s->reordered_input_picture[0];
1419             for (i = 0; i < 4; i++) {
1420                 s->new_picture.f.data[i] += INPLACE_OFFSET;
1421             }
1422         }
1423         ff_copy_picture(&s->current_picture, s->current_picture_ptr);
1424
1425         s->picture_number = s->new_picture.f.display_picture_number;
1426         //printf("dpn:%d\n", s->picture_number);
1427     } else {
1428         memset(&s->new_picture, 0, sizeof(Picture));
1429     }
1430     return 0;
1431 }
1432
1433 int MPV_encode_picture(AVCodecContext *avctx,
1434                        unsigned char *buf, int buf_size, void *data)
1435 {
1436     MpegEncContext *s = avctx->priv_data;
1437     AVFrame *pic_arg  = data;
1438     int i, stuffing_count, context_count = avctx->thread_count;
1439
1440     for (i = 0; i < context_count; i++) {
1441         int start_y = s->thread_context[i]->start_mb_y;
1442         int   end_y = s->thread_context[i]->  end_mb_y;
1443         int h       = s->mb_height;
1444         uint8_t *start = buf + (size_t)(((int64_t) buf_size) * start_y / h);
1445         uint8_t *end   = buf + (size_t)(((int64_t) buf_size) *   end_y / h);
1446
1447         init_put_bits(&s->thread_context[i]->pb, start, end - start);
1448     }
1449
1450     s->picture_in_gop_number++;
1451
1452     if (load_input_picture(s, pic_arg) < 0)
1453         return -1;
1454
1455     if (select_input_picture(s) < 0) {
1456         return -1;
1457     }
1458
1459     /* output? */
1460     if (s->new_picture.f.data[0]) {
1461         s->pict_type = s->new_picture.f.pict_type;
1462         //emms_c();
1463         //printf("qs:%f %f %d\n", s->new_picture.quality,
1464         //       s->current_picture.quality, s->qscale);
1465         MPV_frame_start(s, avctx);
1466 vbv_retry:
1467         if (encode_picture(s, s->picture_number) < 0)
1468             return -1;
1469
1470         avctx->header_bits = s->header_bits;
1471         avctx->mv_bits     = s->mv_bits;
1472         avctx->misc_bits   = s->misc_bits;
1473         avctx->i_tex_bits  = s->i_tex_bits;
1474         avctx->p_tex_bits  = s->p_tex_bits;
1475         avctx->i_count     = s->i_count;
1476         // FIXME f/b_count in avctx
1477         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1478         avctx->skip_count  = s->skip_count;
1479
1480         MPV_frame_end(s);
1481
1482         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1483             ff_mjpeg_encode_picture_trailer(s);
1484
1485         if (avctx->rc_buffer_size) {
1486             RateControlContext *rcc = &s->rc_context;
1487             int max_size = rcc->buffer_index * avctx->rc_max_available_vbv_use;
1488
1489             if (put_bits_count(&s->pb) > max_size &&
1490                 s->lambda < s->avctx->lmax) {
1491                 s->next_lambda = FFMAX(s->lambda + 1, s->lambda *
1492                                        (s->qscale + 1) / s->qscale);
1493                 if (s->adaptive_quant) {
1494                     int i;
1495                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1496                         s->lambda_table[i] =
1497                             FFMAX(s->lambda_table[i] + 1,
1498                                   s->lambda_table[i] * (s->qscale + 1) /
1499                                   s->qscale);
1500                 }
1501                 s->mb_skipped = 0;        // done in MPV_frame_start()
1502                 // done in encode_picture() so we must undo it
1503                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1504                     if (s->flipflop_rounding          ||
1505                         s->codec_id == CODEC_ID_H263P ||
1506                         s->codec_id == CODEC_ID_MPEG4)
1507                         s->no_rounding ^= 1;
1508                 }
1509                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1510                     s->time_base       = s->last_time_base;
1511                     s->last_non_b_time = s->time - s->pp_time;
1512                 }
1513                 //av_log(NULL, AV_LOG_ERROR, "R:%d ", s->next_lambda);
1514                 for (i = 0; i < context_count; i++) {
1515                     PutBitContext *pb = &s->thread_context[i]->pb;
1516                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1517                 }
1518                 goto vbv_retry;
1519             }
1520
1521             assert(s->avctx->rc_max_rate);
1522         }
1523
1524         if (s->flags & CODEC_FLAG_PASS1)
1525             ff_write_pass1_stats(s);
1526
1527         for (i = 0; i < 4; i++) {
1528             s->current_picture_ptr->f.error[i] = s->current_picture.f.error[i];
1529             avctx->error[i] += s->current_picture_ptr->f.error[i];
1530         }
1531
1532         if (s->flags & CODEC_FLAG_PASS1)
1533             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits +
1534                    avctx->i_tex_bits + avctx->p_tex_bits ==
1535                        put_bits_count(&s->pb));
1536         flush_put_bits(&s->pb);
1537         s->frame_bits  = put_bits_count(&s->pb);
1538
1539         stuffing_count = ff_vbv_update(s, s->frame_bits);
1540         if (stuffing_count) {
1541             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1542                     stuffing_count + 50) {
1543                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1544                 return -1;
1545             }
1546
1547             switch (s->codec_id) {
1548             case CODEC_ID_MPEG1VIDEO:
1549             case CODEC_ID_MPEG2VIDEO:
1550                 while (stuffing_count--) {
1551                     put_bits(&s->pb, 8, 0);
1552                 }
1553             break;
1554             case CODEC_ID_MPEG4:
1555                 put_bits(&s->pb, 16, 0);
1556                 put_bits(&s->pb, 16, 0x1C3);
1557                 stuffing_count -= 4;
1558                 while (stuffing_count--) {
1559                     put_bits(&s->pb, 8, 0xFF);
1560                 }
1561             break;
1562             default:
1563                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1564             }
1565             flush_put_bits(&s->pb);
1566             s->frame_bits  = put_bits_count(&s->pb);
1567         }
1568
1569         /* update mpeg1/2 vbv_delay for CBR */
1570         if (s->avctx->rc_max_rate                          &&
1571             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1572             s->out_format == FMT_MPEG1                     &&
1573             90000LL * (avctx->rc_buffer_size - 1) <=
1574                 s->avctx->rc_max_rate * 0xFFFFLL) {
1575             int vbv_delay, min_delay;
1576             double inbits  = s->avctx->rc_max_rate *
1577                              av_q2d(s->avctx->time_base);
1578             int    minbits = s->frame_bits - 8 *
1579                              (s->vbv_delay_ptr - s->pb.buf - 1);
1580             double bits    = s->rc_context.buffer_index + minbits - inbits;
1581
1582             if (bits < 0)
1583                 av_log(s->avctx, AV_LOG_ERROR,
1584                        "Internal error, negative bits\n");
1585
1586             assert(s->repeat_first_field == 0);
1587
1588             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
1589             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
1590                         s->avctx->rc_max_rate;
1591
1592             vbv_delay = FFMAX(vbv_delay, min_delay);
1593
1594             assert(vbv_delay < 0xFFFF);
1595
1596             s->vbv_delay_ptr[0] &= 0xF8;
1597             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
1598             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
1599             s->vbv_delay_ptr[2] &= 0x07;
1600             s->vbv_delay_ptr[2] |= vbv_delay << 3;
1601             avctx->vbv_delay     = vbv_delay * 300;
1602         }
1603         s->total_bits     += s->frame_bits;
1604         avctx->frame_bits  = s->frame_bits;
1605     } else {
1606         assert((put_bits_ptr(&s->pb) == s->pb.buf));
1607         s->frame_bits = 0;
1608     }
1609     assert((s->frame_bits & 7) == 0);
1610
1611     return s->frame_bits / 8;
1612 }
1613
1614 static inline void dct_single_coeff_elimination(MpegEncContext *s,
1615                                                 int n, int threshold)
1616 {
1617     static const char tab[64] = {
1618         3, 2, 2, 1, 1, 1, 1, 1,
1619         1, 1, 1, 1, 1, 1, 1, 1,
1620         1, 1, 1, 1, 1, 1, 1, 1,
1621         0, 0, 0, 0, 0, 0, 0, 0,
1622         0, 0, 0, 0, 0, 0, 0, 0,
1623         0, 0, 0, 0, 0, 0, 0, 0,
1624         0, 0, 0, 0, 0, 0, 0, 0,
1625         0, 0, 0, 0, 0, 0, 0, 0
1626     };
1627     int score = 0;
1628     int run = 0;
1629     int i;
1630     DCTELEM *block = s->block[n];
1631     const int last_index = s->block_last_index[n];
1632     int skip_dc;
1633
1634     if (threshold < 0) {
1635         skip_dc = 0;
1636         threshold = -threshold;
1637     } else
1638         skip_dc = 1;
1639
1640     /* Are all we could set to zero already zero? */
1641     if (last_index <= skip_dc - 1)
1642         return;
1643
1644     for (i = 0; i <= last_index; i++) {
1645         const int j = s->intra_scantable.permutated[i];
1646         const int level = FFABS(block[j]);
1647         if (level == 1) {
1648             if (skip_dc && i == 0)
1649                 continue;
1650             score += tab[run];
1651             run = 0;
1652         } else if (level > 1) {
1653             return;
1654         } else {
1655             run++;
1656         }
1657     }
1658     if (score >= threshold)
1659         return;
1660     for (i = skip_dc; i <= last_index; i++) {
1661         const int j = s->intra_scantable.permutated[i];
1662         block[j] = 0;
1663     }
1664     if (block[0])
1665         s->block_last_index[n] = 0;
1666     else
1667         s->block_last_index[n] = -1;
1668 }
1669
1670 static inline void clip_coeffs(MpegEncContext *s, DCTELEM *block,
1671                                int last_index)
1672 {
1673     int i;
1674     const int maxlevel = s->max_qcoeff;
1675     const int minlevel = s->min_qcoeff;
1676     int overflow = 0;
1677
1678     if (s->mb_intra) {
1679         i = 1; // skip clipping of intra dc
1680     } else
1681         i = 0;
1682
1683     for (; i <= last_index; i++) {
1684         const int j = s->intra_scantable.permutated[i];
1685         int level = block[j];
1686
1687         if (level > maxlevel) {
1688             level = maxlevel;
1689             overflow++;
1690         } else if (level < minlevel) {
1691             level = minlevel;
1692             overflow++;
1693         }
1694
1695         block[j] = level;
1696     }
1697
1698     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1699         av_log(s->avctx, AV_LOG_INFO,
1700                "warning, clipping %d dct coefficients to %d..%d\n",
1701                overflow, minlevel, maxlevel);
1702 }
1703
1704 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
1705 {
1706     int x, y;
1707     // FIXME optimize
1708     for (y = 0; y < 8; y++) {
1709         for (x = 0; x < 8; x++) {
1710             int x2, y2;
1711             int sum = 0;
1712             int sqr = 0;
1713             int count = 0;
1714
1715             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
1716                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
1717                     int v = ptr[x2 + y2 * stride];
1718                     sum += v;
1719                     sqr += v * v;
1720                     count++;
1721                 }
1722             }
1723             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
1724         }
1725     }
1726 }
1727
1728 static av_always_inline void encode_mb_internal(MpegEncContext *s,
1729                                                 int motion_x, int motion_y,
1730                                                 int mb_block_height,
1731                                                 int mb_block_count)
1732 {
1733     int16_t weight[8][64];
1734     DCTELEM orig[8][64];
1735     const int mb_x = s->mb_x;
1736     const int mb_y = s->mb_y;
1737     int i;
1738     int skip_dct[8];
1739     int dct_offset = s->linesize * 8; // default for progressive frames
1740     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1741     int wrap_y, wrap_c;
1742
1743     for (i = 0; i < mb_block_count; i++)
1744         skip_dct[i] = s->skipdct;
1745
1746     if (s->adaptive_quant) {
1747         const int last_qp = s->qscale;
1748         const int mb_xy = mb_x + mb_y * s->mb_stride;
1749
1750         s->lambda = s->lambda_table[mb_xy];
1751         update_qscale(s);
1752
1753         if (!(s->flags & CODEC_FLAG_QP_RD)) {
1754             s->qscale = s->current_picture_ptr->f.qscale_table[mb_xy];
1755             s->dquant = s->qscale - last_qp;
1756
1757             if (s->out_format == FMT_H263) {
1758                 s->dquant = av_clip(s->dquant, -2, 2);
1759
1760                 if (s->codec_id == CODEC_ID_MPEG4) {
1761                     if (!s->mb_intra) {
1762                         if (s->pict_type == AV_PICTURE_TYPE_B) {
1763                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
1764                                 s->dquant = 0;
1765                         }
1766                         if (s->mv_type == MV_TYPE_8X8)
1767                             s->dquant = 0;
1768                     }
1769                 }
1770             }
1771         }
1772         ff_set_qscale(s, last_qp + s->dquant);
1773     } else if (s->flags & CODEC_FLAG_QP_RD)
1774         ff_set_qscale(s, s->qscale + s->dquant);
1775
1776     wrap_y = s->linesize;
1777     wrap_c = s->uvlinesize;
1778     ptr_y  = s->new_picture.f.data[0] +
1779              (mb_y * 16 * wrap_y)              + mb_x * 16;
1780     ptr_cb = s->new_picture.f.data[1] +
1781              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1782     ptr_cr = s->new_picture.f.data[2] +
1783              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1784
1785     if((mb_x*16+16 > s->width || mb_y*16+16 > s->height) && s->codec_id != CODEC_ID_AMV){
1786         uint8_t *ebuf = s->edge_emu_buffer + 32;
1787         s->dsp.emulated_edge_mc(ebuf, ptr_y, wrap_y, 16, 16, mb_x * 16,
1788                                 mb_y * 16, s->width, s->height);
1789         ptr_y = ebuf;
1790         s->dsp.emulated_edge_mc(ebuf + 18 * wrap_y, ptr_cb, wrap_c, 8,
1791                                 mb_block_height, mb_x * 8, mb_y * 8,
1792                                 s->width >> 1, s->height >> 1);
1793         ptr_cb = ebuf + 18 * wrap_y;
1794         s->dsp.emulated_edge_mc(ebuf + 18 * wrap_y + 8, ptr_cr, wrap_c, 8,
1795                                 mb_block_height, mb_x * 8, mb_y * 8,
1796                                 s->width >> 1, s->height >> 1);
1797         ptr_cr = ebuf + 18 * wrap_y + 8;
1798     }
1799
1800     if (s->mb_intra) {
1801         if (s->flags & CODEC_FLAG_INTERLACED_DCT) {
1802             int progressive_score, interlaced_score;
1803
1804             s->interlaced_dct = 0;
1805             progressive_score = s->dsp.ildct_cmp[4](s, ptr_y,
1806                                                     NULL, wrap_y, 8) +
1807                                 s->dsp.ildct_cmp[4](s, ptr_y + wrap_y * 8,
1808                                                     NULL, wrap_y, 8) - 400;
1809
1810             if (progressive_score > 0) {
1811                 interlaced_score = s->dsp.ildct_cmp[4](s, ptr_y,
1812                                                        NULL, wrap_y * 2, 8) +
1813                                    s->dsp.ildct_cmp[4](s, ptr_y + wrap_y,
1814                                                        NULL, wrap_y * 2, 8);
1815                 if (progressive_score > interlaced_score) {
1816                     s->interlaced_dct = 1;
1817
1818                     dct_offset = wrap_y;
1819                     wrap_y <<= 1;
1820                     if (s->chroma_format == CHROMA_422)
1821                         wrap_c <<= 1;
1822                 }
1823             }
1824         }
1825
1826         s->dsp.get_pixels(s->block[0], ptr_y                  , wrap_y);
1827         s->dsp.get_pixels(s->block[1], ptr_y              + 8 , wrap_y);
1828         s->dsp.get_pixels(s->block[2], ptr_y + dct_offset     , wrap_y);
1829         s->dsp.get_pixels(s->block[3], ptr_y + dct_offset + 8 , wrap_y);
1830
1831         if (s->flags & CODEC_FLAG_GRAY) {
1832             skip_dct[4] = 1;
1833             skip_dct[5] = 1;
1834         } else {
1835             s->dsp.get_pixels(s->block[4], ptr_cb, wrap_c);
1836             s->dsp.get_pixels(s->block[5], ptr_cr, wrap_c);
1837             if (!s->chroma_y_shift) { /* 422 */
1838                 s->dsp.get_pixels(s->block[6],
1839                                   ptr_cb + (dct_offset >> 1), wrap_c);
1840                 s->dsp.get_pixels(s->block[7],
1841                                   ptr_cr + (dct_offset >> 1), wrap_c);
1842             }
1843         }
1844     } else {
1845         op_pixels_func (*op_pix)[4];
1846         qpel_mc_func (*op_qpix)[16];
1847         uint8_t *dest_y, *dest_cb, *dest_cr;
1848
1849         dest_y  = s->dest[0];
1850         dest_cb = s->dest[1];
1851         dest_cr = s->dest[2];
1852
1853         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
1854             op_pix  = s->dsp.put_pixels_tab;
1855             op_qpix = s->dsp.put_qpel_pixels_tab;
1856         } else {
1857             op_pix  = s->dsp.put_no_rnd_pixels_tab;
1858             op_qpix = s->dsp.put_no_rnd_qpel_pixels_tab;
1859         }
1860
1861         if (s->mv_dir & MV_DIR_FORWARD) {
1862             MPV_motion(s, dest_y, dest_cb, dest_cr, 0, s->last_picture.f.data,
1863                        op_pix, op_qpix);
1864             op_pix  = s->dsp.avg_pixels_tab;
1865             op_qpix = s->dsp.avg_qpel_pixels_tab;
1866         }
1867         if (s->mv_dir & MV_DIR_BACKWARD) {
1868             MPV_motion(s, dest_y, dest_cb, dest_cr, 1, s->next_picture.f.data,
1869                        op_pix, op_qpix);
1870         }
1871
1872         if (s->flags & CODEC_FLAG_INTERLACED_DCT) {
1873             int progressive_score, interlaced_score;
1874
1875             s->interlaced_dct = 0;
1876             progressive_score = s->dsp.ildct_cmp[0](s, dest_y,
1877                                                     ptr_y,              wrap_y,
1878                                                     8) +
1879                                 s->dsp.ildct_cmp[0](s, dest_y + wrap_y * 8,
1880                                                     ptr_y + wrap_y * 8, wrap_y,
1881                                                     8) - 400;
1882
1883             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
1884                 progressive_score -= 400;
1885
1886             if (progressive_score > 0) {
1887                 interlaced_score = s->dsp.ildct_cmp[0](s, dest_y,
1888                                                        ptr_y,
1889                                                        wrap_y * 2, 8) +
1890                                    s->dsp.ildct_cmp[0](s, dest_y + wrap_y,
1891                                                        ptr_y + wrap_y,
1892                                                        wrap_y * 2, 8);
1893
1894                 if (progressive_score > interlaced_score) {
1895                     s->interlaced_dct = 1;
1896
1897                     dct_offset = wrap_y;
1898                     wrap_y <<= 1;
1899                     if (s->chroma_format == CHROMA_422)
1900                         wrap_c <<= 1;
1901                 }
1902             }
1903         }
1904
1905         s->dsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
1906         s->dsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
1907         s->dsp.diff_pixels(s->block[2], ptr_y + dct_offset,
1908                            dest_y + dct_offset, wrap_y);
1909         s->dsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
1910                            dest_y + dct_offset + 8, wrap_y);
1911
1912         if (s->flags & CODEC_FLAG_GRAY) {
1913             skip_dct[4] = 1;
1914             skip_dct[5] = 1;
1915         } else {
1916             s->dsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
1917             s->dsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
1918             if (!s->chroma_y_shift) { /* 422 */
1919                 s->dsp.diff_pixels(s->block[6], ptr_cb + (dct_offset >> 1),
1920                                    dest_cb + (dct_offset >> 1), wrap_c);
1921                 s->dsp.diff_pixels(s->block[7], ptr_cr + (dct_offset >> 1),
1922                                    dest_cr + (dct_offset >> 1), wrap_c);
1923             }
1924         }
1925         /* pre quantization */
1926         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
1927                 2 * s->qscale * s->qscale) {
1928             // FIXME optimize
1929             if (s->dsp.sad[1](NULL, ptr_y , dest_y,
1930                               wrap_y, 8) < 20 * s->qscale)
1931                 skip_dct[0] = 1;
1932             if (s->dsp.sad[1](NULL, ptr_y + 8,
1933                               dest_y + 8, wrap_y, 8) < 20 * s->qscale)
1934                 skip_dct[1] = 1;
1935             if (s->dsp.sad[1](NULL, ptr_y + dct_offset,
1936                               dest_y + dct_offset, wrap_y, 8) < 20 * s->qscale)
1937                 skip_dct[2] = 1;
1938             if (s->dsp.sad[1](NULL, ptr_y + dct_offset + 8,
1939                               dest_y + dct_offset + 8,
1940                               wrap_y, 8) < 20 * s->qscale)
1941                 skip_dct[3] = 1;
1942             if (s->dsp.sad[1](NULL, ptr_cb, dest_cb,
1943                               wrap_c, 8) < 20 * s->qscale)
1944                 skip_dct[4] = 1;
1945             if (s->dsp.sad[1](NULL, ptr_cr, dest_cr,
1946                               wrap_c, 8) < 20 * s->qscale)
1947                 skip_dct[5] = 1;
1948             if (!s->chroma_y_shift) { /* 422 */
1949                 if (s->dsp.sad[1](NULL, ptr_cb + (dct_offset >> 1),
1950                                   dest_cb + (dct_offset >> 1),
1951                                   wrap_c, 8) < 20 * s->qscale)
1952                     skip_dct[6] = 1;
1953                 if (s->dsp.sad[1](NULL, ptr_cr + (dct_offset >> 1),
1954                                   dest_cr + (dct_offset >> 1),
1955                                   wrap_c, 8) < 20 * s->qscale)
1956                     skip_dct[7] = 1;
1957             }
1958         }
1959     }
1960
1961     if (s->avctx->quantizer_noise_shaping) {
1962         if (!skip_dct[0])
1963             get_visual_weight(weight[0], ptr_y                 , wrap_y);
1964         if (!skip_dct[1])
1965             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
1966         if (!skip_dct[2])
1967             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
1968         if (!skip_dct[3])
1969             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
1970         if (!skip_dct[4])
1971             get_visual_weight(weight[4], ptr_cb                , wrap_c);
1972         if (!skip_dct[5])
1973             get_visual_weight(weight[5], ptr_cr                , wrap_c);
1974         if (!s->chroma_y_shift) { /* 422 */
1975             if (!skip_dct[6])
1976                 get_visual_weight(weight[6], ptr_cb + (dct_offset >> 1),
1977                                   wrap_c);
1978             if (!skip_dct[7])
1979                 get_visual_weight(weight[7], ptr_cr + (dct_offset >> 1),
1980                                   wrap_c);
1981         }
1982         memcpy(orig[0], s->block[0], sizeof(DCTELEM) * 64 * mb_block_count);
1983     }
1984
1985     /* DCT & quantize */
1986     assert(s->out_format != FMT_MJPEG || s->qscale == 8);
1987     {
1988         for (i = 0; i < mb_block_count; i++) {
1989             if (!skip_dct[i]) {
1990                 int overflow;
1991                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
1992                 // FIXME we could decide to change to quantizer instead of
1993                 // clipping
1994                 // JS: I don't think that would be a good idea it could lower
1995                 //     quality instead of improve it. Just INTRADC clipping
1996                 //     deserves changes in quantizer
1997                 if (overflow)
1998                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
1999             } else
2000                 s->block_last_index[i] = -1;
2001         }
2002         if (s->avctx->quantizer_noise_shaping) {
2003             for (i = 0; i < mb_block_count; i++) {
2004                 if (!skip_dct[i]) {
2005                     s->block_last_index[i] =
2006                         dct_quantize_refine(s, s->block[i], weight[i],
2007                                             orig[i], i, s->qscale);
2008                 }
2009             }
2010         }
2011
2012         if (s->luma_elim_threshold && !s->mb_intra)
2013             for (i = 0; i < 4; i++)
2014                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2015         if (s->chroma_elim_threshold && !s->mb_intra)
2016             for (i = 4; i < mb_block_count; i++)
2017                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2018
2019         if (s->flags & CODEC_FLAG_CBP_RD) {
2020             for (i = 0; i < mb_block_count; i++) {
2021                 if (s->block_last_index[i] == -1)
2022                     s->coded_score[i] = INT_MAX / 256;
2023             }
2024         }
2025     }
2026
2027     if ((s->flags & CODEC_FLAG_GRAY) && s->mb_intra) {
2028         s->block_last_index[4] =
2029         s->block_last_index[5] = 0;
2030         s->block[4][0] =
2031         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2032     }
2033
2034     // non c quantize code returns incorrect block_last_index FIXME
2035     if (s->alternate_scan && s->dct_quantize != dct_quantize_c) {
2036         for (i = 0; i < mb_block_count; i++) {
2037             int j;
2038             if (s->block_last_index[i] > 0) {
2039                 for (j = 63; j > 0; j--) {
2040                     if (s->block[i][s->intra_scantable.permutated[j]])
2041                         break;
2042                 }
2043                 s->block_last_index[i] = j;
2044             }
2045         }
2046     }
2047
2048     /* huffman encode */
2049     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2050     case CODEC_ID_MPEG1VIDEO:
2051     case CODEC_ID_MPEG2VIDEO:
2052         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2053             mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2054         break;
2055     case CODEC_ID_MPEG4:
2056         if (CONFIG_MPEG4_ENCODER)
2057             mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2058         break;
2059     case CODEC_ID_MSMPEG4V2:
2060     case CODEC_ID_MSMPEG4V3:
2061     case CODEC_ID_WMV1:
2062         if (CONFIG_MSMPEG4_ENCODER)
2063             msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2064         break;
2065     case CODEC_ID_WMV2:
2066         if (CONFIG_WMV2_ENCODER)
2067             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2068         break;
2069     case CODEC_ID_H261:
2070         if (CONFIG_H261_ENCODER)
2071             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2072         break;
2073     case CODEC_ID_H263:
2074     case CODEC_ID_H263P:
2075     case CODEC_ID_FLV1:
2076     case CODEC_ID_RV10:
2077     case CODEC_ID_RV20:
2078         if (CONFIG_H263_ENCODER)
2079             h263_encode_mb(s, s->block, motion_x, motion_y);
2080         break;
2081     case CODEC_ID_MJPEG:
2082     case CODEC_ID_AMV:
2083         if (CONFIG_MJPEG_ENCODER)
2084             ff_mjpeg_encode_mb(s, s->block);
2085         break;
2086     default:
2087         assert(0);
2088     }
2089 }
2090
2091 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2092 {
2093     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
2094     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
2095 }
2096
2097 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2098     int i;
2099
2100     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2101
2102     /* mpeg1 */
2103     d->mb_skip_run= s->mb_skip_run;
2104     for(i=0; i<3; i++)
2105         d->last_dc[i] = s->last_dc[i];
2106
2107     /* statistics */
2108     d->mv_bits= s->mv_bits;
2109     d->i_tex_bits= s->i_tex_bits;
2110     d->p_tex_bits= s->p_tex_bits;
2111     d->i_count= s->i_count;
2112     d->f_count= s->f_count;
2113     d->b_count= s->b_count;
2114     d->skip_count= s->skip_count;
2115     d->misc_bits= s->misc_bits;
2116     d->last_bits= 0;
2117
2118     d->mb_skipped= 0;
2119     d->qscale= s->qscale;
2120     d->dquant= s->dquant;
2121
2122     d->esc3_level_length= s->esc3_level_length;
2123 }
2124
2125 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2126     int i;
2127
2128     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2129     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2130
2131     /* mpeg1 */
2132     d->mb_skip_run= s->mb_skip_run;
2133     for(i=0; i<3; i++)
2134         d->last_dc[i] = s->last_dc[i];
2135
2136     /* statistics */
2137     d->mv_bits= s->mv_bits;
2138     d->i_tex_bits= s->i_tex_bits;
2139     d->p_tex_bits= s->p_tex_bits;
2140     d->i_count= s->i_count;
2141     d->f_count= s->f_count;
2142     d->b_count= s->b_count;
2143     d->skip_count= s->skip_count;
2144     d->misc_bits= s->misc_bits;
2145
2146     d->mb_intra= s->mb_intra;
2147     d->mb_skipped= s->mb_skipped;
2148     d->mv_type= s->mv_type;
2149     d->mv_dir= s->mv_dir;
2150     d->pb= s->pb;
2151     if(s->data_partitioning){
2152         d->pb2= s->pb2;
2153         d->tex_pb= s->tex_pb;
2154     }
2155     d->block= s->block;
2156     for(i=0; i<8; i++)
2157         d->block_last_index[i]= s->block_last_index[i];
2158     d->interlaced_dct= s->interlaced_dct;
2159     d->qscale= s->qscale;
2160
2161     d->esc3_level_length= s->esc3_level_length;
2162 }
2163
2164 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2165                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2166                            int *dmin, int *next_block, int motion_x, int motion_y)
2167 {
2168     int score;
2169     uint8_t *dest_backup[3];
2170
2171     copy_context_before_encode(s, backup, type);
2172
2173     s->block= s->blocks[*next_block];
2174     s->pb= pb[*next_block];
2175     if(s->data_partitioning){
2176         s->pb2   = pb2   [*next_block];
2177         s->tex_pb= tex_pb[*next_block];
2178     }
2179
2180     if(*next_block){
2181         memcpy(dest_backup, s->dest, sizeof(s->dest));
2182         s->dest[0] = s->rd_scratchpad;
2183         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
2184         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
2185         assert(s->linesize >= 32); //FIXME
2186     }
2187
2188     encode_mb(s, motion_x, motion_y);
2189
2190     score= put_bits_count(&s->pb);
2191     if(s->data_partitioning){
2192         score+= put_bits_count(&s->pb2);
2193         score+= put_bits_count(&s->tex_pb);
2194     }
2195
2196     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2197         MPV_decode_mb(s, s->block);
2198
2199         score *= s->lambda2;
2200         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2201     }
2202
2203     if(*next_block){
2204         memcpy(s->dest, dest_backup, sizeof(s->dest));
2205     }
2206
2207     if(score<*dmin){
2208         *dmin= score;
2209         *next_block^=1;
2210
2211         copy_context_after_encode(best, s, type);
2212     }
2213 }
2214
2215 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2216     uint32_t *sq = ff_squareTbl + 256;
2217     int acc=0;
2218     int x,y;
2219
2220     if(w==16 && h==16)
2221         return s->dsp.sse[0](NULL, src1, src2, stride, 16);
2222     else if(w==8 && h==8)
2223         return s->dsp.sse[1](NULL, src1, src2, stride, 8);
2224
2225     for(y=0; y<h; y++){
2226         for(x=0; x<w; x++){
2227             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2228         }
2229     }
2230
2231     assert(acc>=0);
2232
2233     return acc;
2234 }
2235
2236 static int sse_mb(MpegEncContext *s){
2237     int w= 16;
2238     int h= 16;
2239
2240     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2241     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2242
2243     if(w==16 && h==16)
2244       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2245         return  s->dsp.nsse[0](s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2246                +s->dsp.nsse[1](s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2247                +s->dsp.nsse[1](s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2248       }else{
2249         return  s->dsp.sse[0](NULL, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2250                +s->dsp.sse[1](NULL, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2251                +s->dsp.sse[1](NULL, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2252       }
2253     else
2254         return  sse(s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2255                +sse(s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2256                +sse(s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2257 }
2258
2259 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2260     MpegEncContext *s= *(void**)arg;
2261
2262
2263     s->me.pre_pass=1;
2264     s->me.dia_size= s->avctx->pre_dia_size;
2265     s->first_slice_line=1;
2266     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2267         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2268             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2269         }
2270         s->first_slice_line=0;
2271     }
2272
2273     s->me.pre_pass=0;
2274
2275     return 0;
2276 }
2277
2278 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2279     MpegEncContext *s= *(void**)arg;
2280
2281     ff_check_alignment();
2282
2283     s->me.dia_size= s->avctx->dia_size;
2284     s->first_slice_line=1;
2285     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2286         s->mb_x=0; //for block init below
2287         ff_init_block_index(s);
2288         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2289             s->block_index[0]+=2;
2290             s->block_index[1]+=2;
2291             s->block_index[2]+=2;
2292             s->block_index[3]+=2;
2293
2294             /* compute motion vector & mb_type and store in context */
2295             if(s->pict_type==AV_PICTURE_TYPE_B)
2296                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2297             else
2298                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2299         }
2300         s->first_slice_line=0;
2301     }
2302     return 0;
2303 }
2304
2305 static int mb_var_thread(AVCodecContext *c, void *arg){
2306     MpegEncContext *s= *(void**)arg;
2307     int mb_x, mb_y;
2308
2309     ff_check_alignment();
2310
2311     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2312         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2313             int xx = mb_x * 16;
2314             int yy = mb_y * 16;
2315             uint8_t *pix = s->new_picture.f.data[0] + (yy * s->linesize) + xx;
2316             int varc;
2317             int sum = s->dsp.pix_sum(pix, s->linesize);
2318
2319             varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)sum*sum)>>8) + 500 + 128)>>8;
2320
2321             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2322             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2323             s->me.mb_var_sum_temp    += varc;
2324         }
2325     }
2326     return 0;
2327 }
2328
2329 static void write_slice_end(MpegEncContext *s){
2330     if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4){
2331         if(s->partitioned_frame){
2332             ff_mpeg4_merge_partitions(s);
2333         }
2334
2335         ff_mpeg4_stuffing(&s->pb);
2336     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2337         ff_mjpeg_encode_stuffing(&s->pb);
2338     }
2339
2340     avpriv_align_put_bits(&s->pb);
2341     flush_put_bits(&s->pb);
2342
2343     if((s->flags&CODEC_FLAG_PASS1) && !s->partitioned_frame)
2344         s->misc_bits+= get_bits_diff(s);
2345 }
2346
2347 static int encode_thread(AVCodecContext *c, void *arg){
2348     MpegEncContext *s= *(void**)arg;
2349     int mb_x, mb_y, pdif = 0;
2350     int chr_h= 16>>s->chroma_y_shift;
2351     int i, j;
2352     MpegEncContext best_s, backup_s;
2353     uint8_t bit_buf[2][MAX_MB_BYTES];
2354     uint8_t bit_buf2[2][MAX_MB_BYTES];
2355     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2356     PutBitContext pb[2], pb2[2], tex_pb[2];
2357 //printf("%d->%d\n", s->resync_mb_y, s->end_mb_y);
2358
2359     ff_check_alignment();
2360
2361     for(i=0; i<2; i++){
2362         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2363         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2364         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2365     }
2366
2367     s->last_bits= put_bits_count(&s->pb);
2368     s->mv_bits=0;
2369     s->misc_bits=0;
2370     s->i_tex_bits=0;
2371     s->p_tex_bits=0;
2372     s->i_count=0;
2373     s->f_count=0;
2374     s->b_count=0;
2375     s->skip_count=0;
2376
2377     for(i=0; i<3; i++){
2378         /* init last dc values */
2379         /* note: quant matrix value (8) is implied here */
2380         s->last_dc[i] = 128 << s->intra_dc_precision;
2381
2382         s->current_picture.f.error[i] = 0;
2383     }
2384     if(s->codec_id==CODEC_ID_AMV){
2385         s->last_dc[0] = 128*8/13;
2386         s->last_dc[1] = 128*8/14;
2387         s->last_dc[2] = 128*8/14;
2388     }
2389     s->mb_skip_run = 0;
2390     memset(s->last_mv, 0, sizeof(s->last_mv));
2391
2392     s->last_mv_dir = 0;
2393
2394     switch(s->codec_id){
2395     case CODEC_ID_H263:
2396     case CODEC_ID_H263P:
2397     case CODEC_ID_FLV1:
2398         if (CONFIG_H263_ENCODER)
2399             s->gob_index = ff_h263_get_gob_height(s);
2400         break;
2401     case CODEC_ID_MPEG4:
2402         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2403             ff_mpeg4_init_partitions(s);
2404         break;
2405     }
2406
2407     s->resync_mb_x=0;
2408     s->resync_mb_y=0;
2409     s->first_slice_line = 1;
2410     s->ptr_lastgob = s->pb.buf;
2411     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2412 //    printf("row %d at %X\n", s->mb_y, (int)s);
2413         s->mb_x=0;
2414         s->mb_y= mb_y;
2415
2416         ff_set_qscale(s, s->qscale);
2417         ff_init_block_index(s);
2418
2419         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2420             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2421             int mb_type= s->mb_type[xy];
2422 //            int d;
2423             int dmin= INT_MAX;
2424             int dir;
2425
2426             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2427                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2428                 return -1;
2429             }
2430             if(s->data_partitioning){
2431                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2432                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2433                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2434                     return -1;
2435                 }
2436             }
2437
2438             s->mb_x = mb_x;
2439             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2440             ff_update_block_index(s);
2441
2442             if(CONFIG_H261_ENCODER && s->codec_id == CODEC_ID_H261){
2443                 ff_h261_reorder_mb_index(s);
2444                 xy= s->mb_y*s->mb_stride + s->mb_x;
2445                 mb_type= s->mb_type[xy];
2446             }
2447
2448             /* write gob / video packet header  */
2449             if(s->rtp_mode){
2450                 int current_packet_size, is_gob_start;
2451
2452                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2453
2454                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2455
2456                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2457
2458                 switch(s->codec_id){
2459                 case CODEC_ID_H263:
2460                 case CODEC_ID_H263P:
2461                     if(!s->h263_slice_structured)
2462                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2463                     break;
2464                 case CODEC_ID_MPEG2VIDEO:
2465                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2466                 case CODEC_ID_MPEG1VIDEO:
2467                     if(s->mb_skip_run) is_gob_start=0;
2468                     break;
2469                 }
2470
2471                 if(is_gob_start){
2472                     if(s->start_mb_y != mb_y || mb_x!=0){
2473                         write_slice_end(s);
2474
2475                         if(CONFIG_MPEG4_ENCODER && s->codec_id==CODEC_ID_MPEG4 && s->partitioned_frame){
2476                             ff_mpeg4_init_partitions(s);
2477                         }
2478                     }
2479
2480                     assert((put_bits_count(&s->pb)&7) == 0);
2481                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2482
2483                     if(s->avctx->error_rate && s->resync_mb_x + s->resync_mb_y > 0){
2484                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2485                         int d= 100 / s->avctx->error_rate;
2486                         if(r % d == 0){
2487                             current_packet_size=0;
2488                             s->pb.buf_ptr= s->ptr_lastgob;
2489                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2490                         }
2491                     }
2492
2493                     if (s->avctx->rtp_callback){
2494                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2495                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2496                     }
2497
2498                     switch(s->codec_id){
2499                     case CODEC_ID_MPEG4:
2500                         if (CONFIG_MPEG4_ENCODER) {
2501                             ff_mpeg4_encode_video_packet_header(s);
2502                             ff_mpeg4_clean_buffers(s);
2503                         }
2504                     break;
2505                     case CODEC_ID_MPEG1VIDEO:
2506                     case CODEC_ID_MPEG2VIDEO:
2507                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2508                             ff_mpeg1_encode_slice_header(s);
2509                             ff_mpeg1_clean_buffers(s);
2510                         }
2511                     break;
2512                     case CODEC_ID_H263:
2513                     case CODEC_ID_H263P:
2514                         if (CONFIG_H263_ENCODER)
2515                             h263_encode_gob_header(s, mb_y);
2516                     break;
2517                     }
2518
2519                     if(s->flags&CODEC_FLAG_PASS1){
2520                         int bits= put_bits_count(&s->pb);
2521                         s->misc_bits+= bits - s->last_bits;
2522                         s->last_bits= bits;
2523                     }
2524
2525                     s->ptr_lastgob += current_packet_size;
2526                     s->first_slice_line=1;
2527                     s->resync_mb_x=mb_x;
2528                     s->resync_mb_y=mb_y;
2529                 }
2530             }
2531
2532             if(  (s->resync_mb_x   == s->mb_x)
2533                && s->resync_mb_y+1 == s->mb_y){
2534                 s->first_slice_line=0;
2535             }
2536
2537             s->mb_skipped=0;
2538             s->dquant=0; //only for QP_RD
2539
2540             if(mb_type & (mb_type-1) || (s->flags & CODEC_FLAG_QP_RD)){ // more than 1 MB type possible or CODEC_FLAG_QP_RD
2541                 int next_block=0;
2542                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2543
2544                 copy_context_before_encode(&backup_s, s, -1);
2545                 backup_s.pb= s->pb;
2546                 best_s.data_partitioning= s->data_partitioning;
2547                 best_s.partitioned_frame= s->partitioned_frame;
2548                 if(s->data_partitioning){
2549                     backup_s.pb2= s->pb2;
2550                     backup_s.tex_pb= s->tex_pb;
2551                 }
2552
2553                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2554                     s->mv_dir = MV_DIR_FORWARD;
2555                     s->mv_type = MV_TYPE_16X16;
2556                     s->mb_intra= 0;
2557                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2558                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2559                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2560                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2561                 }
2562                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2563                     s->mv_dir = MV_DIR_FORWARD;
2564                     s->mv_type = MV_TYPE_FIELD;
2565                     s->mb_intra= 0;
2566                     for(i=0; i<2; i++){
2567                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2568                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2569                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2570                     }
2571                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2572                                  &dmin, &next_block, 0, 0);
2573                 }
2574                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2575                     s->mv_dir = MV_DIR_FORWARD;
2576                     s->mv_type = MV_TYPE_16X16;
2577                     s->mb_intra= 0;
2578                     s->mv[0][0][0] = 0;
2579                     s->mv[0][0][1] = 0;
2580                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2581                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2582                 }
2583                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2584                     s->mv_dir = MV_DIR_FORWARD;
2585                     s->mv_type = MV_TYPE_8X8;
2586                     s->mb_intra= 0;
2587                     for(i=0; i<4; i++){
2588                         s->mv[0][i][0] = s->current_picture.f.motion_val[0][s->block_index[i]][0];
2589                         s->mv[0][i][1] = s->current_picture.f.motion_val[0][s->block_index[i]][1];
2590                     }
2591                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2592                                  &dmin, &next_block, 0, 0);
2593                 }
2594                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2595                     s->mv_dir = MV_DIR_FORWARD;
2596                     s->mv_type = MV_TYPE_16X16;
2597                     s->mb_intra= 0;
2598                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2599                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2600                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2601                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2602                 }
2603                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2604                     s->mv_dir = MV_DIR_BACKWARD;
2605                     s->mv_type = MV_TYPE_16X16;
2606                     s->mb_intra= 0;
2607                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2608                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2609                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2610                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2611                 }
2612                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2613                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2614                     s->mv_type = MV_TYPE_16X16;
2615                     s->mb_intra= 0;
2616                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2617                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2618                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2619                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2620                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2621                                  &dmin, &next_block, 0, 0);
2622                 }
2623                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2624                     s->mv_dir = MV_DIR_FORWARD;
2625                     s->mv_type = MV_TYPE_FIELD;
2626                     s->mb_intra= 0;
2627                     for(i=0; i<2; i++){
2628                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2629                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2630                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2631                     }
2632                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2633                                  &dmin, &next_block, 0, 0);
2634                 }
2635                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2636                     s->mv_dir = MV_DIR_BACKWARD;
2637                     s->mv_type = MV_TYPE_FIELD;
2638                     s->mb_intra= 0;
2639                     for(i=0; i<2; i++){
2640                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2641                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2642                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2643                     }
2644                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2645                                  &dmin, &next_block, 0, 0);
2646                 }
2647                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2648                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2649                     s->mv_type = MV_TYPE_FIELD;
2650                     s->mb_intra= 0;
2651                     for(dir=0; dir<2; dir++){
2652                         for(i=0; i<2; i++){
2653                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2654                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2655                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2656                         }
2657                     }
2658                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2659                                  &dmin, &next_block, 0, 0);
2660                 }
2661                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2662                     s->mv_dir = 0;
2663                     s->mv_type = MV_TYPE_16X16;
2664                     s->mb_intra= 1;
2665                     s->mv[0][0][0] = 0;
2666                     s->mv[0][0][1] = 0;
2667                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2668                                  &dmin, &next_block, 0, 0);
2669                     if(s->h263_pred || s->h263_aic){
2670                         if(best_s.mb_intra)
2671                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2672                         else
2673                             ff_clean_intra_table_entries(s); //old mode?
2674                     }
2675                 }
2676
2677                 if((s->flags & CODEC_FLAG_QP_RD) && dmin < INT_MAX){
2678                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2679                         const int last_qp= backup_s.qscale;
2680                         int qpi, qp, dc[6];
2681                         DCTELEM ac[6][16];
2682                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2683                         static const int dquant_tab[4]={-1,1,-2,2};
2684
2685                         assert(backup_s.dquant == 0);
2686
2687                         //FIXME intra
2688                         s->mv_dir= best_s.mv_dir;
2689                         s->mv_type = MV_TYPE_16X16;
2690                         s->mb_intra= best_s.mb_intra;
2691                         s->mv[0][0][0] = best_s.mv[0][0][0];
2692                         s->mv[0][0][1] = best_s.mv[0][0][1];
2693                         s->mv[1][0][0] = best_s.mv[1][0][0];
2694                         s->mv[1][0][1] = best_s.mv[1][0][1];
2695
2696                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
2697                         for(; qpi<4; qpi++){
2698                             int dquant= dquant_tab[qpi];
2699                             qp= last_qp + dquant;
2700                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2701                                 continue;
2702                             backup_s.dquant= dquant;
2703                             if(s->mb_intra && s->dc_val[0]){
2704                                 for(i=0; i<6; i++){
2705                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2706                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(DCTELEM)*16);
2707                                 }
2708                             }
2709
2710                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2711                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2712                             if(best_s.qscale != qp){
2713                                 if(s->mb_intra && s->dc_val[0]){
2714                                     for(i=0; i<6; i++){
2715                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2716                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(DCTELEM)*16);
2717                                     }
2718                                 }
2719                             }
2720                         }
2721                     }
2722                 }
2723                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2724                     int mx= s->b_direct_mv_table[xy][0];
2725                     int my= s->b_direct_mv_table[xy][1];
2726
2727                     backup_s.dquant = 0;
2728                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2729                     s->mb_intra= 0;
2730                     ff_mpeg4_set_direct_mv(s, mx, my);
2731                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2732                                  &dmin, &next_block, mx, my);
2733                 }
2734                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2735                     backup_s.dquant = 0;
2736                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2737                     s->mb_intra= 0;
2738                     ff_mpeg4_set_direct_mv(s, 0, 0);
2739                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2740                                  &dmin, &next_block, 0, 0);
2741                 }
2742                 if(!best_s.mb_intra && s->flags2&CODEC_FLAG2_SKIP_RD){
2743                     int coded=0;
2744                     for(i=0; i<6; i++)
2745                         coded |= s->block_last_index[i];
2746                     if(coded){
2747                         int mx,my;
2748                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2749                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
2750                             mx=my=0; //FIXME find the one we actually used
2751                             ff_mpeg4_set_direct_mv(s, mx, my);
2752                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2753                             mx= s->mv[1][0][0];
2754                             my= s->mv[1][0][1];
2755                         }else{
2756                             mx= s->mv[0][0][0];
2757                             my= s->mv[0][0][1];
2758                         }
2759
2760                         s->mv_dir= best_s.mv_dir;
2761                         s->mv_type = best_s.mv_type;
2762                         s->mb_intra= 0;
2763 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2764                         s->mv[0][0][1] = best_s.mv[0][0][1];
2765                         s->mv[1][0][0] = best_s.mv[1][0][0];
2766                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2767                         backup_s.dquant= 0;
2768                         s->skipdct=1;
2769                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2770                                         &dmin, &next_block, mx, my);
2771                         s->skipdct=0;
2772                     }
2773                 }
2774
2775                 s->current_picture.f.qscale_table[xy] = best_s.qscale;
2776
2777                 copy_context_after_encode(s, &best_s, -1);
2778
2779                 pb_bits_count= put_bits_count(&s->pb);
2780                 flush_put_bits(&s->pb);
2781                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2782                 s->pb= backup_s.pb;
2783
2784                 if(s->data_partitioning){
2785                     pb2_bits_count= put_bits_count(&s->pb2);
2786                     flush_put_bits(&s->pb2);
2787                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2788                     s->pb2= backup_s.pb2;
2789
2790                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2791                     flush_put_bits(&s->tex_pb);
2792                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2793                     s->tex_pb= backup_s.tex_pb;
2794                 }
2795                 s->last_bits= put_bits_count(&s->pb);
2796
2797                 if (CONFIG_H263_ENCODER &&
2798                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2799                     ff_h263_update_motion_val(s);
2800
2801                 if(next_block==0){ //FIXME 16 vs linesize16
2802                     s->dsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
2803                     s->dsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2804                     s->dsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2805                 }
2806
2807                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
2808                     MPV_decode_mb(s, s->block);
2809             } else {
2810                 int motion_x = 0, motion_y = 0;
2811                 s->mv_type=MV_TYPE_16X16;
2812                 // only one MB-Type possible
2813
2814                 switch(mb_type){
2815                 case CANDIDATE_MB_TYPE_INTRA:
2816                     s->mv_dir = 0;
2817                     s->mb_intra= 1;
2818                     motion_x= s->mv[0][0][0] = 0;
2819                     motion_y= s->mv[0][0][1] = 0;
2820                     break;
2821                 case CANDIDATE_MB_TYPE_INTER:
2822                     s->mv_dir = MV_DIR_FORWARD;
2823                     s->mb_intra= 0;
2824                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
2825                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
2826                     break;
2827                 case CANDIDATE_MB_TYPE_INTER_I:
2828                     s->mv_dir = MV_DIR_FORWARD;
2829                     s->mv_type = MV_TYPE_FIELD;
2830                     s->mb_intra= 0;
2831                     for(i=0; i<2; i++){
2832                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2833                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2834                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2835                     }
2836                     break;
2837                 case CANDIDATE_MB_TYPE_INTER4V:
2838                     s->mv_dir = MV_DIR_FORWARD;
2839                     s->mv_type = MV_TYPE_8X8;
2840                     s->mb_intra= 0;
2841                     for(i=0; i<4; i++){
2842                         s->mv[0][i][0] = s->current_picture.f.motion_val[0][s->block_index[i]][0];
2843                         s->mv[0][i][1] = s->current_picture.f.motion_val[0][s->block_index[i]][1];
2844                     }
2845                     break;
2846                 case CANDIDATE_MB_TYPE_DIRECT:
2847                     if (CONFIG_MPEG4_ENCODER) {
2848                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2849                         s->mb_intra= 0;
2850                         motion_x=s->b_direct_mv_table[xy][0];
2851                         motion_y=s->b_direct_mv_table[xy][1];
2852                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
2853                     }
2854                     break;
2855                 case CANDIDATE_MB_TYPE_DIRECT0:
2856                     if (CONFIG_MPEG4_ENCODER) {
2857                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2858                         s->mb_intra= 0;
2859                         ff_mpeg4_set_direct_mv(s, 0, 0);
2860                     }
2861                     break;
2862                 case CANDIDATE_MB_TYPE_BIDIR:
2863                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2864                     s->mb_intra= 0;
2865                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2866                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2867                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2868                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2869                     break;
2870                 case CANDIDATE_MB_TYPE_BACKWARD:
2871                     s->mv_dir = MV_DIR_BACKWARD;
2872                     s->mb_intra= 0;
2873                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2874                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2875                     break;
2876                 case CANDIDATE_MB_TYPE_FORWARD:
2877                     s->mv_dir = MV_DIR_FORWARD;
2878                     s->mb_intra= 0;
2879                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2880                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2881 //                    printf(" %d %d ", motion_x, motion_y);
2882                     break;
2883                 case CANDIDATE_MB_TYPE_FORWARD_I:
2884                     s->mv_dir = MV_DIR_FORWARD;
2885                     s->mv_type = MV_TYPE_FIELD;
2886                     s->mb_intra= 0;
2887                     for(i=0; i<2; i++){
2888                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2889                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2890                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2891                     }
2892                     break;
2893                 case CANDIDATE_MB_TYPE_BACKWARD_I:
2894                     s->mv_dir = MV_DIR_BACKWARD;
2895                     s->mv_type = MV_TYPE_FIELD;
2896                     s->mb_intra= 0;
2897                     for(i=0; i<2; i++){
2898                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2899                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2900                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2901                     }
2902                     break;
2903                 case CANDIDATE_MB_TYPE_BIDIR_I:
2904                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2905                     s->mv_type = MV_TYPE_FIELD;
2906                     s->mb_intra= 0;
2907                     for(dir=0; dir<2; dir++){
2908                         for(i=0; i<2; i++){
2909                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2910                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2911                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2912                         }
2913                     }
2914                     break;
2915                 default:
2916                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
2917                 }
2918
2919                 encode_mb(s, motion_x, motion_y);
2920
2921                 // RAL: Update last macroblock type
2922                 s->last_mv_dir = s->mv_dir;
2923
2924                 if (CONFIG_H263_ENCODER &&
2925                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2926                     ff_h263_update_motion_val(s);
2927
2928                 MPV_decode_mb(s, s->block);
2929             }
2930
2931             /* clean the MV table in IPS frames for direct mode in B frames */
2932             if(s->mb_intra /* && I,P,S_TYPE */){
2933                 s->p_mv_table[xy][0]=0;
2934                 s->p_mv_table[xy][1]=0;
2935             }
2936
2937             if(s->flags&CODEC_FLAG_PSNR){
2938                 int w= 16;
2939                 int h= 16;
2940
2941                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2942                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2943
2944                 s->current_picture.f.error[0] += sse(
2945                     s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
2946                     s->dest[0], w, h, s->linesize);
2947                 s->current_picture.f.error[1] += sse(
2948                     s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2949                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2950                 s->current_picture.f.error[2] += sse(
2951                     s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2952                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2953             }
2954             if(s->loop_filter){
2955                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
2956                     ff_h263_loop_filter(s);
2957             }
2958 //printf("MB %d %d bits\n", s->mb_x+s->mb_y*s->mb_stride, put_bits_count(&s->pb));
2959         }
2960     }
2961
2962     //not beautiful here but we must write it before flushing so it has to be here
2963     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
2964         msmpeg4_encode_ext_header(s);
2965
2966     write_slice_end(s);
2967
2968     /* Send the last GOB if RTP */
2969     if (s->avctx->rtp_callback) {
2970         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
2971         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
2972         /* Call the RTP callback to send the last GOB */
2973         emms_c();
2974         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
2975     }
2976
2977     return 0;
2978 }
2979
2980 #define MERGE(field) dst->field += src->field; src->field=0
2981 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
2982     MERGE(me.scene_change_score);
2983     MERGE(me.mc_mb_var_sum_temp);
2984     MERGE(me.mb_var_sum_temp);
2985 }
2986
2987 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
2988     int i;
2989
2990     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
2991     MERGE(dct_count[1]);
2992     MERGE(mv_bits);
2993     MERGE(i_tex_bits);
2994     MERGE(p_tex_bits);
2995     MERGE(i_count);
2996     MERGE(f_count);
2997     MERGE(b_count);
2998     MERGE(skip_count);
2999     MERGE(misc_bits);
3000     MERGE(error_count);
3001     MERGE(padding_bug_score);
3002     MERGE(current_picture.f.error[0]);
3003     MERGE(current_picture.f.error[1]);
3004     MERGE(current_picture.f.error[2]);
3005
3006     if(dst->avctx->noise_reduction){
3007         for(i=0; i<64; i++){
3008             MERGE(dct_error_sum[0][i]);
3009             MERGE(dct_error_sum[1][i]);
3010         }
3011     }
3012
3013     assert(put_bits_count(&src->pb) % 8 ==0);
3014     assert(put_bits_count(&dst->pb) % 8 ==0);
3015     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3016     flush_put_bits(&dst->pb);
3017 }
3018
3019 static int estimate_qp(MpegEncContext *s, int dry_run){
3020     if (s->next_lambda){
3021         s->current_picture_ptr->f.quality =
3022         s->current_picture.f.quality = s->next_lambda;
3023         if(!dry_run) s->next_lambda= 0;
3024     } else if (!s->fixed_qscale) {
3025         s->current_picture_ptr->f.quality =
3026         s->current_picture.f.quality = ff_rate_estimate_qscale(s, dry_run);
3027         if (s->current_picture.f.quality < 0)
3028             return -1;
3029     }
3030
3031     if(s->adaptive_quant){
3032         switch(s->codec_id){
3033         case CODEC_ID_MPEG4:
3034             if (CONFIG_MPEG4_ENCODER)
3035                 ff_clean_mpeg4_qscales(s);
3036             break;
3037         case CODEC_ID_H263:
3038         case CODEC_ID_H263P:
3039         case CODEC_ID_FLV1:
3040             if (CONFIG_H263_ENCODER)
3041                 ff_clean_h263_qscales(s);
3042             break;
3043         default:
3044             ff_init_qscale_tab(s);
3045         }
3046
3047         s->lambda= s->lambda_table[0];
3048         //FIXME broken
3049     }else
3050         s->lambda = s->current_picture.f.quality;
3051 //printf("%d %d\n", s->avctx->global_quality, s->current_picture.quality);
3052     update_qscale(s);
3053     return 0;
3054 }
3055
3056 /* must be called before writing the header */
3057 static void set_frame_distances(MpegEncContext * s){
3058     assert(s->current_picture_ptr->f.pts != AV_NOPTS_VALUE);
3059     s->time = s->current_picture_ptr->f.pts * s->avctx->time_base.num;
3060
3061     if(s->pict_type==AV_PICTURE_TYPE_B){
3062         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3063         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3064     }else{
3065         s->pp_time= s->time - s->last_non_b_time;
3066         s->last_non_b_time= s->time;
3067         assert(s->picture_number==0 || s->pp_time > 0);
3068     }
3069 }
3070
3071 static int encode_picture(MpegEncContext *s, int picture_number)
3072 {
3073     int i;
3074     int bits;
3075     int context_count = s->avctx->thread_count;
3076
3077     s->picture_number = picture_number;
3078
3079     /* Reset the average MB variance */
3080     s->me.mb_var_sum_temp    =
3081     s->me.mc_mb_var_sum_temp = 0;
3082
3083     /* we need to initialize some time vars before we can encode b-frames */
3084     // RAL: Condition added for MPEG1VIDEO
3085     if (s->codec_id == CODEC_ID_MPEG1VIDEO || s->codec_id == CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3086         set_frame_distances(s);
3087     if(CONFIG_MPEG4_ENCODER && s->codec_id == CODEC_ID_MPEG4)
3088         ff_set_mpeg4_time(s);
3089
3090     s->me.scene_change_score=0;
3091
3092 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3093
3094     if(s->pict_type==AV_PICTURE_TYPE_I){
3095         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3096         else                        s->no_rounding=0;
3097     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3098         if(s->flipflop_rounding || s->codec_id == CODEC_ID_H263P || s->codec_id == CODEC_ID_MPEG4)
3099             s->no_rounding ^= 1;
3100     }
3101
3102     if(s->flags & CODEC_FLAG_PASS2){
3103         if (estimate_qp(s,1) < 0)
3104             return -1;
3105         ff_get_2pass_fcode(s);
3106     }else if(!(s->flags & CODEC_FLAG_QSCALE)){
3107         if(s->pict_type==AV_PICTURE_TYPE_B)
3108             s->lambda= s->last_lambda_for[s->pict_type];
3109         else
3110             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3111         update_qscale(s);
3112     }
3113
3114     if(s->codec_id != CODEC_ID_AMV){
3115         if(s->q_chroma_intra_matrix   != s->q_intra_matrix  ) av_freep(&s->q_chroma_intra_matrix);
3116         if(s->q_chroma_intra_matrix16 != s->q_intra_matrix16) av_freep(&s->q_chroma_intra_matrix16);
3117         s->q_chroma_intra_matrix   = s->q_intra_matrix;
3118         s->q_chroma_intra_matrix16 = s->q_intra_matrix16;
3119     }
3120
3121     s->mb_intra=0; //for the rate distortion & bit compare functions
3122     for(i=1; i<context_count; i++){
3123         ff_update_duplicate_context(s->thread_context[i], s);
3124     }
3125
3126     if(ff_init_me(s)<0)
3127         return -1;
3128
3129     /* Estimate motion for every MB */
3130     if(s->pict_type != AV_PICTURE_TYPE_I){
3131         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
3132         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
3133         if(s->pict_type != AV_PICTURE_TYPE_B && s->avctx->me_threshold==0){
3134             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
3135                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3136             }
3137         }
3138
3139         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3140     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3141         /* I-Frame */
3142         for(i=0; i<s->mb_stride*s->mb_height; i++)
3143             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3144
3145         if(!s->fixed_qscale){
3146             /* finding spatial complexity for I-frame rate control */
3147             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3148         }
3149     }
3150     for(i=1; i<context_count; i++){
3151         merge_context_after_me(s, s->thread_context[i]);
3152     }
3153     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3154     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3155     emms_c();
3156
3157     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
3158         s->pict_type= AV_PICTURE_TYPE_I;
3159         for(i=0; i<s->mb_stride*s->mb_height; i++)
3160             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3161 //printf("Scene change detected, encoding as I Frame %d %d\n", s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3162     }
3163
3164     if(!s->umvplus){
3165         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3166             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3167
3168             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3169                 int a,b;
3170                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3171                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3172                 s->f_code= FFMAX3(s->f_code, a, b);
3173             }
3174
3175             ff_fix_long_p_mvs(s);
3176             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3177             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3178                 int j;
3179                 for(i=0; i<2; i++){
3180                     for(j=0; j<2; j++)
3181                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3182                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3183                 }
3184             }
3185         }
3186
3187         if(s->pict_type==AV_PICTURE_TYPE_B){
3188             int a, b;
3189
3190             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3191             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3192             s->f_code = FFMAX(a, b);
3193
3194             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3195             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3196             s->b_code = FFMAX(a, b);
3197
3198             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3199             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3200             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3201             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3202             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3203                 int dir, j;
3204                 for(dir=0; dir<2; dir++){
3205                     for(i=0; i<2; i++){
3206                         for(j=0; j<2; j++){
3207                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3208                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3209                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3210                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3211                         }
3212                     }
3213                 }
3214             }
3215         }
3216     }
3217
3218     if (estimate_qp(s, 0) < 0)
3219         return -1;
3220
3221     if(s->qscale < 3 && s->max_qcoeff<=128 && s->pict_type==AV_PICTURE_TYPE_I && !(s->flags & CODEC_FLAG_QSCALE))
3222         s->qscale= 3; //reduce clipping problems
3223
3224     if (s->out_format == FMT_MJPEG) {
3225         /* for mjpeg, we do include qscale in the matrix */
3226         for(i=1;i<64;i++){
3227             int j= s->dsp.idct_permutation[i];
3228
3229             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
3230         }
3231         s->y_dc_scale_table=
3232         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3233         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3234         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
3235                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3236         s->qscale= 8;
3237     }
3238     if(s->codec_id == CODEC_ID_AMV){
3239         static const uint8_t y[32]={13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13};
3240         static const uint8_t c[32]={14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14};
3241         for(i=1;i<64;i++){
3242             int j= s->dsp.idct_permutation[ff_zigzag_direct[i]];
3243
3244             s->intra_matrix[j] = sp5x_quant_table[5*2+0][i];
3245             s->chroma_intra_matrix[j] = sp5x_quant_table[5*2+1][i];
3246         }
3247         s->y_dc_scale_table= y;
3248         s->c_dc_scale_table= c;
3249         s->intra_matrix[0] = 13;
3250         s->chroma_intra_matrix[0] = 14;
3251         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
3252                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3253         ff_convert_matrix(&s->dsp, s->q_chroma_intra_matrix, s->q_chroma_intra_matrix16,
3254                        s->chroma_intra_matrix, s->intra_quant_bias, 8, 8, 1);
3255         s->qscale= 8;
3256     }
3257
3258     //FIXME var duplication
3259     s->current_picture_ptr->f.key_frame =
3260     s->current_picture.f.key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3261     s->current_picture_ptr->f.pict_type =
3262     s->current_picture.f.pict_type = s->pict_type;
3263
3264     if (s->current_picture.f.key_frame)
3265         s->picture_in_gop_number=0;
3266
3267     s->last_bits= put_bits_count(&s->pb);
3268     switch(s->out_format) {
3269     case FMT_MJPEG:
3270         if (CONFIG_MJPEG_ENCODER)
3271             ff_mjpeg_encode_picture_header(s);
3272         break;
3273     case FMT_H261:
3274         if (CONFIG_H261_ENCODER)
3275             ff_h261_encode_picture_header(s, picture_number);
3276         break;
3277     case FMT_H263:
3278         if (CONFIG_WMV2_ENCODER && s->codec_id == CODEC_ID_WMV2)
3279             ff_wmv2_encode_picture_header(s, picture_number);
3280         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3281             msmpeg4_encode_picture_header(s, picture_number);
3282         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
3283             mpeg4_encode_picture_header(s, picture_number);
3284         else if (CONFIG_RV10_ENCODER && s->codec_id == CODEC_ID_RV10)
3285             rv10_encode_picture_header(s, picture_number);
3286         else if (CONFIG_RV20_ENCODER && s->codec_id == CODEC_ID_RV20)
3287             rv20_encode_picture_header(s, picture_number);
3288         else if (CONFIG_FLV_ENCODER && s->codec_id == CODEC_ID_FLV1)
3289             ff_flv_encode_picture_header(s, picture_number);
3290         else if (CONFIG_H263_ENCODER)
3291             h263_encode_picture_header(s, picture_number);
3292         break;
3293     case FMT_MPEG1:
3294         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
3295             mpeg1_encode_picture_header(s, picture_number);
3296         break;
3297     case FMT_H264:
3298         break;
3299     default:
3300         assert(0);
3301     }
3302     bits= put_bits_count(&s->pb);
3303     s->header_bits= bits - s->last_bits;
3304
3305     for(i=1; i<context_count; i++){
3306         update_duplicate_context_after_me(s->thread_context[i], s);
3307     }
3308     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3309     for(i=1; i<context_count; i++){
3310         merge_context_after_encode(s, s->thread_context[i]);
3311     }
3312     emms_c();
3313     return 0;
3314 }
3315
3316 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block){
3317     const int intra= s->mb_intra;
3318     int i;
3319
3320     s->dct_count[intra]++;
3321
3322     for(i=0; i<64; i++){
3323         int level= block[i];
3324
3325         if(level){
3326             if(level>0){
3327                 s->dct_error_sum[intra][i] += level;
3328                 level -= s->dct_offset[intra][i];
3329                 if(level<0) level=0;
3330             }else{
3331                 s->dct_error_sum[intra][i] -= level;
3332                 level += s->dct_offset[intra][i];
3333                 if(level>0) level=0;
3334             }
3335             block[i]= level;
3336         }
3337     }
3338 }
3339
3340 static int dct_quantize_trellis_c(MpegEncContext *s,
3341                                   DCTELEM *block, int n,
3342                                   int qscale, int *overflow){
3343     const int *qmat;
3344     const uint8_t *scantable= s->intra_scantable.scantable;
3345     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3346     int max=0;
3347     unsigned int threshold1, threshold2;
3348     int bias=0;
3349     int run_tab[65];
3350     int level_tab[65];
3351     int score_tab[65];
3352     int survivor[65];
3353     int survivor_count;
3354     int last_run=0;
3355     int last_level=0;
3356     int last_score= 0;
3357     int last_i;
3358     int coeff[2][64];
3359     int coeff_count[64];
3360     int qmul, qadd, start_i, last_non_zero, i, dc;
3361     const int esc_length= s->ac_esc_length;
3362     uint8_t * length;
3363     uint8_t * last_length;
3364     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3365
3366     s->dsp.fdct (block);
3367
3368     if(s->dct_error_sum)
3369         s->denoise_dct(s, block);
3370     qmul= qscale*16;
3371     qadd= ((qscale-1)|1)*8;
3372
3373     if (s->mb_intra) {
3374         int q;
3375         if (!s->h263_aic) {
3376             if (n < 4)
3377                 q = s->y_dc_scale;
3378             else
3379                 q = s->c_dc_scale;
3380             q = q << 3;
3381         } else{
3382             /* For AIC we skip quant/dequant of INTRADC */
3383             q = 1 << 3;
3384             qadd=0;
3385         }
3386
3387         /* note: block[0] is assumed to be positive */
3388         block[0] = (block[0] + (q >> 1)) / q;
3389         start_i = 1;
3390         last_non_zero = 0;
3391         qmat = n < 4 ? s->q_intra_matrix[qscale] : s->q_chroma_intra_matrix[qscale];
3392         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3393             bias= 1<<(QMAT_SHIFT-1);
3394         length     = s->intra_ac_vlc_length;
3395         last_length= s->intra_ac_vlc_last_length;
3396     } else {
3397         start_i = 0;
3398         last_non_zero = -1;
3399         qmat = s->q_inter_matrix[qscale];
3400         length     = s->inter_ac_vlc_length;
3401         last_length= s->inter_ac_vlc_last_length;
3402     }
3403     last_i= start_i;
3404
3405     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3406     threshold2= (threshold1<<1);
3407
3408     for(i=63; i>=start_i; i--) {
3409         const int j = scantable[i];
3410         int level = block[j] * qmat[j];
3411
3412         if(((unsigned)(level+threshold1))>threshold2){
3413             last_non_zero = i;
3414             break;
3415         }
3416     }
3417
3418     for(i=start_i; i<=last_non_zero; i++) {
3419         const int j = scantable[i];
3420         int level = block[j] * qmat[j];
3421
3422 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3423 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3424         if(((unsigned)(level+threshold1))>threshold2){
3425             if(level>0){
3426                 level= (bias + level)>>QMAT_SHIFT;
3427                 coeff[0][i]= level;
3428                 coeff[1][i]= level-1;
3429 //                coeff[2][k]= level-2;
3430             }else{
3431                 level= (bias - level)>>QMAT_SHIFT;
3432                 coeff[0][i]= -level;
3433                 coeff[1][i]= -level+1;
3434 //                coeff[2][k]= -level+2;
3435             }
3436             coeff_count[i]= FFMIN(level, 2);
3437             assert(coeff_count[i]);
3438             max |=level;
3439         }else{
3440             coeff[0][i]= (level>>31)|1;
3441             coeff_count[i]= 1;
3442         }
3443     }
3444
3445     *overflow= s->max_qcoeff < max; //overflow might have happened
3446
3447     if(last_non_zero < start_i){
3448         memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3449         return last_non_zero;
3450     }
3451
3452     score_tab[start_i]= 0;
3453     survivor[0]= start_i;
3454     survivor_count= 1;
3455
3456     for(i=start_i; i<=last_non_zero; i++){
3457         int level_index, j, zero_distortion;
3458         int dct_coeff= FFABS(block[ scantable[i] ]);
3459         int best_score=256*256*256*120;
3460
3461         if (   s->dsp.fdct == fdct_ifast
3462 #ifndef FAAN_POSTSCALE
3463             || s->dsp.fdct == ff_faandct
3464 #endif
3465            )
3466             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3467         zero_distortion= dct_coeff*dct_coeff;
3468
3469         for(level_index=0; level_index < coeff_count[i]; level_index++){
3470             int distortion;
3471             int level= coeff[level_index][i];
3472             const int alevel= FFABS(level);
3473             int unquant_coeff;
3474
3475             assert(level);
3476
3477             if(s->out_format == FMT_H263){
3478                 unquant_coeff= alevel*qmul + qadd;
3479             }else{ //MPEG1
3480                 j= s->dsp.idct_permutation[ scantable[i] ]; //FIXME optimize
3481                 if(s->mb_intra){
3482                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3483                         unquant_coeff =   (unquant_coeff - 1) | 1;
3484                 }else{
3485                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3486                         unquant_coeff =   (unquant_coeff - 1) | 1;
3487                 }
3488                 unquant_coeff<<= 3;
3489             }
3490
3491             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3492             level+=64;
3493             if((level&(~127)) == 0){
3494                 for(j=survivor_count-1; j>=0; j--){
3495                     int run= i - survivor[j];
3496                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3497                     score += score_tab[i-run];
3498
3499                     if(score < best_score){
3500                         best_score= score;
3501                         run_tab[i+1]= run;
3502                         level_tab[i+1]= level-64;
3503                     }
3504                 }
3505
3506                 if(s->out_format == FMT_H263){
3507                     for(j=survivor_count-1; j>=0; j--){
3508                         int run= i - survivor[j];
3509                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3510                         score += score_tab[i-run];
3511                         if(score < last_score){
3512                             last_score= score;
3513                             last_run= run;
3514                             last_level= level-64;
3515                             last_i= i+1;
3516                         }
3517                     }
3518                 }
3519             }else{
3520                 distortion += esc_length*lambda;
3521                 for(j=survivor_count-1; j>=0; j--){
3522                     int run= i - survivor[j];
3523                     int score= distortion + score_tab[i-run];
3524
3525                     if(score < best_score){
3526                         best_score= score;
3527                         run_tab[i+1]= run;
3528                         level_tab[i+1]= level-64;
3529                     }
3530                 }
3531
3532                 if(s->out_format == FMT_H263){
3533                   for(j=survivor_count-1; j>=0; j--){
3534                         int run= i - survivor[j];
3535                         int score= distortion + score_tab[i-run];
3536                         if(score < last_score){
3537                             last_score= score;
3538                             last_run= run;
3539                             last_level= level-64;
3540                             last_i= i+1;
3541                         }
3542                     }
3543                 }
3544             }
3545         }
3546
3547         score_tab[i+1]= best_score;
3548
3549         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
3550         if(last_non_zero <= 27){
3551             for(; survivor_count; survivor_count--){
3552                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3553                     break;
3554             }
3555         }else{
3556             for(; survivor_count; survivor_count--){
3557                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3558                     break;
3559             }
3560         }
3561
3562         survivor[ survivor_count++ ]= i+1;
3563     }
3564
3565     if(s->out_format != FMT_H263){
3566         last_score= 256*256*256*120;
3567         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3568             int score= score_tab[i];
3569             if(i) score += lambda*2; //FIXME exacter?
3570
3571             if(score < last_score){
3572                 last_score= score;
3573                 last_i= i;
3574                 last_level= level_tab[i];
3575                 last_run= run_tab[i];
3576             }
3577         }
3578     }
3579
3580     s->coded_score[n] = last_score;
3581
3582     dc= FFABS(block[0]);
3583     last_non_zero= last_i - 1;
3584     memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3585
3586     if(last_non_zero < start_i)
3587         return last_non_zero;
3588
3589     if(last_non_zero == 0 && start_i == 0){
3590         int best_level= 0;
3591         int best_score= dc * dc;
3592
3593         for(i=0; i<coeff_count[0]; i++){
3594             int level= coeff[i][0];
3595             int alevel= FFABS(level);
3596             int unquant_coeff, score, distortion;
3597
3598             if(s->out_format == FMT_H263){
3599                     unquant_coeff= (alevel*qmul + qadd)>>3;
3600             }else{ //MPEG1
3601                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3602                     unquant_coeff =   (unquant_coeff - 1) | 1;
3603             }
3604             unquant_coeff = (unquant_coeff + 4) >> 3;
3605             unquant_coeff<<= 3 + 3;
3606
3607             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3608             level+=64;
3609             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3610             else                    score= distortion + esc_length*lambda;
3611
3612             if(score < best_score){
3613                 best_score= score;
3614                 best_level= level - 64;
3615             }
3616         }
3617         block[0]= best_level;
3618         s->coded_score[n] = best_score - dc*dc;
3619         if(best_level == 0) return -1;
3620         else                return last_non_zero;
3621     }
3622
3623     i= last_i;
3624     assert(last_level);
3625
3626     block[ perm_scantable[last_non_zero] ]= last_level;
3627     i -= last_run + 1;
3628
3629     for(; i>start_i; i -= run_tab[i] + 1){
3630         block[ perm_scantable[i-1] ]= level_tab[i];
3631     }
3632
3633     return last_non_zero;
3634 }
3635
3636 //#define REFINE_STATS 1
3637 static int16_t basis[64][64];
3638
3639 static void build_basis(uint8_t *perm){
3640     int i, j, x, y;
3641     emms_c();
3642     for(i=0; i<8; i++){
3643         for(j=0; j<8; j++){
3644             for(y=0; y<8; y++){
3645                 for(x=0; x<8; x++){
3646                     double s= 0.25*(1<<BASIS_SHIFT);
3647                     int index= 8*i + j;
3648                     int perm_index= perm[index];
3649                     if(i==0) s*= sqrt(0.5);
3650                     if(j==0) s*= sqrt(0.5);
3651                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3652                 }
3653             }
3654         }
3655     }
3656 }
3657
3658 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3659                         DCTELEM *block, int16_t *weight, DCTELEM *orig,
3660                         int n, int qscale){
3661     int16_t rem[64];
3662     LOCAL_ALIGNED_16(DCTELEM, d1, [64]);
3663     const uint8_t *scantable= s->intra_scantable.scantable;
3664     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3665 //    unsigned int threshold1, threshold2;
3666 //    int bias=0;
3667     int run_tab[65];
3668     int prev_run=0;
3669     int prev_level=0;
3670     int qmul, qadd, start_i, last_non_zero, i, dc;
3671     uint8_t * length;
3672     uint8_t * last_length;
3673     int lambda;
3674     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3675 #ifdef REFINE_STATS
3676 static int count=0;
3677 static int after_last=0;
3678 static int to_zero=0;
3679 static int from_zero=0;
3680 static int raise=0;
3681 static int lower=0;
3682 static int messed_sign=0;
3683 #endif
3684
3685     if(basis[0][0] == 0)
3686         build_basis(s->dsp.idct_permutation);
3687
3688     qmul= qscale*2;
3689     qadd= (qscale-1)|1;
3690     if (s->mb_intra) {
3691         if (!s->h263_aic) {
3692             if (n < 4)
3693                 q = s->y_dc_scale;
3694             else
3695                 q = s->c_dc_scale;
3696         } else{
3697             /* For AIC we skip quant/dequant of INTRADC */
3698             q = 1;
3699             qadd=0;
3700         }
3701         q <<= RECON_SHIFT-3;
3702         /* note: block[0] is assumed to be positive */
3703         dc= block[0]*q;
3704 //        block[0] = (block[0] + (q >> 1)) / q;
3705         start_i = 1;
3706 //        if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3707 //            bias= 1<<(QMAT_SHIFT-1);
3708         length     = s->intra_ac_vlc_length;
3709         last_length= s->intra_ac_vlc_last_length;
3710     } else {
3711         dc= 0;
3712         start_i = 0;
3713         length     = s->inter_ac_vlc_length;
3714         last_length= s->inter_ac_vlc_last_length;
3715     }
3716     last_non_zero = s->block_last_index[n];
3717
3718 #ifdef REFINE_STATS
3719 {START_TIMER
3720 #endif
3721     dc += (1<<(RECON_SHIFT-1));
3722     for(i=0; i<64; i++){
3723         rem[i]= dc - (orig[i]<<RECON_SHIFT); //FIXME  use orig dirrectly instead of copying to rem[]
3724     }
3725 #ifdef REFINE_STATS
3726 STOP_TIMER("memset rem[]")}
3727 #endif
3728     sum=0;
3729     for(i=0; i<64; i++){
3730         int one= 36;
3731         int qns=4;
3732         int w;
3733
3734         w= FFABS(weight[i]) + qns*one;
3735         w= 15 + (48*qns*one + w/2)/w; // 16 .. 63
3736
3737         weight[i] = w;
3738 //        w=weight[i] = (63*qns + (w/2)) / w;
3739
3740         assert(w>0);
3741         assert(w<(1<<6));
3742         sum += w*w;
3743     }
3744     lambda= sum*(uint64_t)s->lambda2 >> (FF_LAMBDA_SHIFT - 6 + 6 + 6 + 6);
3745 #ifdef REFINE_STATS
3746 {START_TIMER
3747 #endif
3748     run=0;
3749     rle_index=0;
3750     for(i=start_i; i<=last_non_zero; i++){
3751         int j= perm_scantable[i];
3752         const int level= block[j];
3753         int coeff;
3754
3755         if(level){
3756             if(level<0) coeff= qmul*level - qadd;
3757             else        coeff= qmul*level + qadd;
3758             run_tab[rle_index++]=run;
3759             run=0;
3760
3761             s->dsp.add_8x8basis(rem, basis[j], coeff);
3762         }else{
3763             run++;
3764         }
3765     }
3766 #ifdef REFINE_STATS
3767 if(last_non_zero>0){
3768 STOP_TIMER("init rem[]")
3769 }
3770 }
3771
3772 {START_TIMER
3773 #endif
3774     for(;;){
3775         int best_score=s->dsp.try_8x8basis(rem, weight, basis[0], 0);
3776         int best_coeff=0;
3777         int best_change=0;
3778         int run2, best_unquant_change=0, analyze_gradient;
3779 #ifdef REFINE_STATS
3780 {START_TIMER
3781 #endif
3782         analyze_gradient = last_non_zero > 2 || s->avctx->quantizer_noise_shaping >= 3;
3783
3784         if(analyze_gradient){
3785 #ifdef REFINE_STATS
3786 {START_TIMER
3787 #endif
3788             for(i=0; i<64; i++){
3789                 int w= weight[i];
3790
3791                 d1[i] = (rem[i]*w*w + (1<<(RECON_SHIFT+12-1)))>>(RECON_SHIFT+12);
3792             }
3793 #ifdef REFINE_STATS
3794 STOP_TIMER("rem*w*w")}
3795 {START_TIMER
3796 #endif
3797             s->dsp.fdct(d1);
3798 #ifdef REFINE_STATS
3799 STOP_TIMER("dct")}
3800 #endif
3801         }
3802
3803         if(start_i){
3804             const int level= block[0];
3805             int change, old_coeff;
3806
3807             assert(s->mb_intra);
3808
3809             old_coeff= q*level;
3810
3811             for(change=-1; change<=1; change+=2){
3812                 int new_level= level + change;
3813                 int score, new_coeff;
3814
3815                 new_coeff= q*new_level;
3816                 if(new_coeff >= 2048 || new_coeff < 0)
3817                     continue;
3818
3819                 score= s->dsp.try_8x8basis(rem, weight, basis[0], new_coeff - old_coeff);
3820                 if(score<best_score){
3821                     best_score= score;
3822                     best_coeff= 0;
3823                     best_change= change;
3824                     best_unquant_change= new_coeff - old_coeff;
3825                 }
3826             }
3827         }
3828
3829         run=0;
3830         rle_index=0;
3831         run2= run_tab[rle_index++];
3832         prev_level=0;
3833         prev_run=0;
3834
3835         for(i=start_i; i<64; i++){
3836             int j= perm_scantable[i];
3837             const int level= block[j];
3838             int change, old_coeff;
3839
3840             if(s->avctx->quantizer_noise_shaping < 3 && i > last_non_zero + 1)
3841                 break;
3842
3843             if(level){
3844                 if(level<0) old_coeff= qmul*level - qadd;
3845                 else        old_coeff= qmul*level + qadd;
3846                 run2= run_tab[rle_index++]; //FIXME ! maybe after last
3847             }else{
3848                 old_coeff=0;
3849                 run2--;
3850                 assert(run2>=0 || i >= last_non_zero );
3851             }
3852
3853             for(change=-1; change<=1; change+=2){
3854                 int new_level= level + change;