65f22817da2876f507fd599c9897d4dc4f01ad6a
[ffmpeg.git] / libavcodec / mpegvideo_enc.c
1 /*
2  * The simplest mpeg encoder (well, it was the simplest!)
3  * Copyright (c) 2000,2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of Libav.
9  *
10  * Libav is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * Libav is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with Libav; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * The simplest mpeg encoder (well, it was the simplest!).
28  */
29
30 #include "libavutil/intmath.h"
31 #include "libavutil/mathematics.h"
32 #include "libavutil/opt.h"
33 #include "avcodec.h"
34 #include "dsputil.h"
35 #include "mpegvideo.h"
36 #include "h263.h"
37 #include "mjpegenc.h"
38 #include "msmpeg4.h"
39 #include "faandct.h"
40 #include "thread.h"
41 #include "aandcttab.h"
42 #include "flv.h"
43 #include "mpeg4video.h"
44 #include "internal.h"
45 #include "bytestream.h"
46 #include <limits.h>
47
48 //#undef NDEBUG
49 //#include <assert.h>
50
51 static int encode_picture(MpegEncContext *s, int picture_number);
52 static int dct_quantize_refine(MpegEncContext *s, DCTELEM *block, int16_t *weight, DCTELEM *orig, int n, int qscale);
53 static int sse_mb(MpegEncContext *s);
54 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block);
55 static int dct_quantize_trellis_c(MpegEncContext *s, DCTELEM *block, int n, int qscale, int *overflow);
56
57 /* enable all paranoid tests for rounding, overflows, etc... */
58 //#define PARANOID
59
60 //#define DEBUG
61
62 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
63 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
64
65 const AVOption ff_mpv_generic_options[] = {
66     FF_MPV_COMMON_OPTS
67     { NULL },
68 };
69
70 void ff_convert_matrix(DSPContext *dsp, int (*qmat)[64],
71                        uint16_t (*qmat16)[2][64],
72                        const uint16_t *quant_matrix,
73                        int bias, int qmin, int qmax, int intra)
74 {
75     int qscale;
76     int shift = 0;
77
78     for (qscale = qmin; qscale <= qmax; qscale++) {
79         int i;
80         if (dsp->fdct == ff_jpeg_fdct_islow_8 ||
81             dsp->fdct == ff_jpeg_fdct_islow_10 ||
82             dsp->fdct == ff_faandct) {
83             for (i = 0; i < 64; i++) {
84                 const int j = dsp->idct_permutation[i];
85                 /* 16 <= qscale * quant_matrix[i] <= 7905
86                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
87                  *             19952 <=              x  <= 249205026
88                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
89                  *           3444240 >= (1 << 36) / (x) >= 275 */
90
91                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
92                                         (qscale * quant_matrix[j]));
93             }
94         } else if (dsp->fdct == ff_fdct_ifast) {
95             for (i = 0; i < 64; i++) {
96                 const int j = dsp->idct_permutation[i];
97                 /* 16 <= qscale * quant_matrix[i] <= 7905
98                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
99                  *             19952 <=              x  <= 249205026
100                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
101                  *           3444240 >= (1 << 36) / (x) >= 275 */
102
103                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) /
104                                         (ff_aanscales[i] * qscale *
105                                          quant_matrix[j]));
106             }
107         } else {
108             for (i = 0; i < 64; i++) {
109                 const int j = dsp->idct_permutation[i];
110                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
111                  * Assume x = qscale * quant_matrix[i]
112                  * So             16 <=              x  <= 7905
113                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
114                  * so          32768 >= (1 << 19) / (x) >= 67 */
115                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) /
116                                         (qscale * quant_matrix[j]));
117                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
118                 //                    (qscale * quant_matrix[i]);
119                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) /
120                                        (qscale * quant_matrix[j]);
121
122                 if (qmat16[qscale][0][i] == 0 ||
123                     qmat16[qscale][0][i] == 128 * 256)
124                     qmat16[qscale][0][i] = 128 * 256 - 1;
125                 qmat16[qscale][1][i] =
126                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
127                                 qmat16[qscale][0][i]);
128             }
129         }
130
131         for (i = intra; i < 64; i++) {
132             int64_t max = 8191;
133             if (dsp->fdct == ff_fdct_ifast) {
134                 max = (8191LL * ff_aanscales[i]) >> 14;
135             }
136             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
137                 shift++;
138             }
139         }
140     }
141     if (shift) {
142         av_log(NULL, AV_LOG_INFO,
143                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
144                QMAT_SHIFT - shift);
145     }
146 }
147
148 static inline void update_qscale(MpegEncContext *s)
149 {
150     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
151                 (FF_LAMBDA_SHIFT + 7);
152     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
153
154     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
155                  FF_LAMBDA_SHIFT;
156 }
157
158 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
159 {
160     int i;
161
162     if (matrix) {
163         put_bits(pb, 1, 1);
164         for (i = 0; i < 64; i++) {
165             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
166         }
167     } else
168         put_bits(pb, 1, 0);
169 }
170
171 /**
172  * init s->current_picture.qscale_table from s->lambda_table
173  */
174 void ff_init_qscale_tab(MpegEncContext *s)
175 {
176     int8_t * const qscale_table = s->current_picture.f.qscale_table;
177     int i;
178
179     for (i = 0; i < s->mb_num; i++) {
180         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
181         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
182         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
183                                                   s->avctx->qmax);
184     }
185 }
186
187 static void copy_picture_attributes(MpegEncContext *s,
188                                     AVFrame *dst,
189                                     AVFrame *src)
190 {
191     int i;
192
193     dst->pict_type              = src->pict_type;
194     dst->quality                = src->quality;
195     dst->coded_picture_number   = src->coded_picture_number;
196     dst->display_picture_number = src->display_picture_number;
197     //dst->reference              = src->reference;
198     dst->pts                    = src->pts;
199     dst->interlaced_frame       = src->interlaced_frame;
200     dst->top_field_first        = src->top_field_first;
201
202     if (s->avctx->me_threshold) {
203         if (!src->motion_val[0])
204             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.motion_val not set!\n");
205         if (!src->mb_type)
206             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.mb_type not set!\n");
207         if (!src->ref_index[0])
208             av_log(s->avctx, AV_LOG_ERROR, "AVFrame.ref_index not set!\n");
209         if (src->motion_subsample_log2 != dst->motion_subsample_log2)
210             av_log(s->avctx, AV_LOG_ERROR,
211                    "AVFrame.motion_subsample_log2 doesn't match! (%d!=%d)\n",
212                    src->motion_subsample_log2, dst->motion_subsample_log2);
213
214         memcpy(dst->mb_type, src->mb_type,
215                s->mb_stride * s->mb_height * sizeof(dst->mb_type[0]));
216
217         for (i = 0; i < 2; i++) {
218             int stride = ((16 * s->mb_width ) >>
219                           src->motion_subsample_log2) + 1;
220             int height = ((16 * s->mb_height) >> src->motion_subsample_log2);
221
222             if (src->motion_val[i] &&
223                 src->motion_val[i] != dst->motion_val[i]) {
224                 memcpy(dst->motion_val[i], src->motion_val[i],
225                        2 * stride * height * sizeof(int16_t));
226             }
227             if (src->ref_index[i] && src->ref_index[i] != dst->ref_index[i]) {
228                 memcpy(dst->ref_index[i], src->ref_index[i],
229                        s->mb_stride * 4 * s->mb_height * sizeof(int8_t));
230             }
231         }
232     }
233 }
234
235 static void update_duplicate_context_after_me(MpegEncContext *dst,
236                                               MpegEncContext *src)
237 {
238 #define COPY(a) dst->a= src->a
239     COPY(pict_type);
240     COPY(current_picture);
241     COPY(f_code);
242     COPY(b_code);
243     COPY(qscale);
244     COPY(lambda);
245     COPY(lambda2);
246     COPY(picture_in_gop_number);
247     COPY(gop_picture_number);
248     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
249     COPY(progressive_frame);    // FIXME don't set in encode_header
250     COPY(partitioned_frame);    // FIXME don't set in encode_header
251 #undef COPY
252 }
253
254 /**
255  * Set the given MpegEncContext to defaults for encoding.
256  * the changed fields will not depend upon the prior state of the MpegEncContext.
257  */
258 static void MPV_encode_defaults(MpegEncContext *s)
259 {
260     int i;
261     ff_MPV_common_defaults(s);
262
263     for (i = -16; i < 16; i++) {
264         default_fcode_tab[i + MAX_MV] = 1;
265     }
266     s->me.mv_penalty = default_mv_penalty;
267     s->fcode_tab     = default_fcode_tab;
268 }
269
270 /* init video encoder */
271 av_cold int ff_MPV_encode_init(AVCodecContext *avctx)
272 {
273     MpegEncContext *s = avctx->priv_data;
274     int i;
275     int chroma_h_shift, chroma_v_shift;
276
277     MPV_encode_defaults(s);
278
279     switch (avctx->codec_id) {
280     case AV_CODEC_ID_MPEG2VIDEO:
281         if (avctx->pix_fmt != PIX_FMT_YUV420P &&
282             avctx->pix_fmt != PIX_FMT_YUV422P) {
283             av_log(avctx, AV_LOG_ERROR,
284                    "only YUV420 and YUV422 are supported\n");
285             return -1;
286         }
287         break;
288     case AV_CODEC_ID_LJPEG:
289         if (avctx->pix_fmt != PIX_FMT_YUVJ420P &&
290             avctx->pix_fmt != PIX_FMT_YUVJ422P &&
291             avctx->pix_fmt != PIX_FMT_YUVJ444P &&
292             avctx->pix_fmt != PIX_FMT_BGRA     &&
293             ((avctx->pix_fmt != PIX_FMT_YUV420P &&
294               avctx->pix_fmt != PIX_FMT_YUV422P &&
295               avctx->pix_fmt != PIX_FMT_YUV444P) ||
296              avctx->strict_std_compliance > FF_COMPLIANCE_UNOFFICIAL)) {
297             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in LJPEG\n");
298             return -1;
299         }
300         break;
301     case AV_CODEC_ID_MJPEG:
302         if (avctx->pix_fmt != PIX_FMT_YUVJ420P &&
303             avctx->pix_fmt != PIX_FMT_YUVJ422P &&
304             ((avctx->pix_fmt != PIX_FMT_YUV420P &&
305               avctx->pix_fmt != PIX_FMT_YUV422P) ||
306              avctx->strict_std_compliance > FF_COMPLIANCE_UNOFFICIAL)) {
307             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
308             return -1;
309         }
310         break;
311     default:
312         if (avctx->pix_fmt != PIX_FMT_YUV420P) {
313             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
314             return -1;
315         }
316     }
317
318     switch (avctx->pix_fmt) {
319     case PIX_FMT_YUVJ422P:
320     case PIX_FMT_YUV422P:
321         s->chroma_format = CHROMA_422;
322         break;
323     case PIX_FMT_YUVJ420P:
324     case PIX_FMT_YUV420P:
325     default:
326         s->chroma_format = CHROMA_420;
327         break;
328     }
329
330     s->bit_rate = avctx->bit_rate;
331     s->width    = avctx->width;
332     s->height   = avctx->height;
333     if (avctx->gop_size > 600 &&
334         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
335         av_log(avctx, AV_LOG_ERROR,
336                "Warning keyframe interval too large! reducing it ...\n");
337         avctx->gop_size = 600;
338     }
339     s->gop_size     = avctx->gop_size;
340     s->avctx        = avctx;
341     s->flags        = avctx->flags;
342     s->flags2       = avctx->flags2;
343     s->max_b_frames = avctx->max_b_frames;
344     s->codec_id     = avctx->codec->id;
345 #if FF_API_MPV_GLOBAL_OPTS
346     if (avctx->luma_elim_threshold)
347         s->luma_elim_threshold   = avctx->luma_elim_threshold;
348     if (avctx->chroma_elim_threshold)
349         s->chroma_elim_threshold = avctx->chroma_elim_threshold;
350 #endif
351     s->strict_std_compliance = avctx->strict_std_compliance;
352     s->quarter_sample     = (avctx->flags & CODEC_FLAG_QPEL) != 0;
353     s->mpeg_quant         = avctx->mpeg_quant;
354     s->rtp_mode           = !!avctx->rtp_payload_size;
355     s->intra_dc_precision = avctx->intra_dc_precision;
356     s->user_specified_pts = AV_NOPTS_VALUE;
357
358     if (s->gop_size <= 1) {
359         s->intra_only = 1;
360         s->gop_size   = 12;
361     } else {
362         s->intra_only = 0;
363     }
364
365     s->me_method = avctx->me_method;
366
367     /* Fixed QSCALE */
368     s->fixed_qscale = !!(avctx->flags & CODEC_FLAG_QSCALE);
369
370 #if FF_API_MPV_GLOBAL_OPTS
371     if (s->flags & CODEC_FLAG_QP_RD)
372         s->mpv_flags |= FF_MPV_FLAG_QP_RD;
373 #endif
374
375     s->adaptive_quant = (s->avctx->lumi_masking ||
376                          s->avctx->dark_masking ||
377                          s->avctx->temporal_cplx_masking ||
378                          s->avctx->spatial_cplx_masking  ||
379                          s->avctx->p_masking      ||
380                          s->avctx->border_masking ||
381                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
382                         !s->fixed_qscale;
383
384     s->loop_filter      = !!(s->flags & CODEC_FLAG_LOOP_FILTER);
385
386     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
387         av_log(avctx, AV_LOG_ERROR,
388                "a vbv buffer size is needed, "
389                "for encoding with a maximum bitrate\n");
390         return -1;
391     }
392
393     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
394         av_log(avctx, AV_LOG_INFO,
395                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
396     }
397
398     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
399         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
400         return -1;
401     }
402
403     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
404         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
405         return -1;
406     }
407
408     if (avctx->rc_max_rate &&
409         avctx->rc_max_rate == avctx->bit_rate &&
410         avctx->rc_max_rate != avctx->rc_min_rate) {
411         av_log(avctx, AV_LOG_INFO,
412                "impossible bitrate constraints, this will fail\n");
413     }
414
415     if (avctx->rc_buffer_size &&
416         avctx->bit_rate * (int64_t)avctx->time_base.num >
417             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
418         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
419         return -1;
420     }
421
422     if (!s->fixed_qscale &&
423         avctx->bit_rate * av_q2d(avctx->time_base) >
424             avctx->bit_rate_tolerance) {
425         av_log(avctx, AV_LOG_ERROR,
426                "bitrate tolerance too small for bitrate\n");
427         return -1;
428     }
429
430     if (s->avctx->rc_max_rate &&
431         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
432         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
433          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
434         90000LL * (avctx->rc_buffer_size - 1) >
435             s->avctx->rc_max_rate * 0xFFFFLL) {
436         av_log(avctx, AV_LOG_INFO,
437                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
438                "specified vbv buffer is too large for the given bitrate!\n");
439     }
440
441     if ((s->flags & CODEC_FLAG_4MV)  && s->codec_id != AV_CODEC_ID_MPEG4 &&
442         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
443         s->codec_id != AV_CODEC_ID_FLV1) {
444         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
445         return -1;
446     }
447
448     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
449         av_log(avctx, AV_LOG_ERROR,
450                "OBMC is only supported with simple mb decision\n");
451         return -1;
452     }
453
454     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
455         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
456         return -1;
457     }
458
459     if (s->max_b_frames                    &&
460         s->codec_id != AV_CODEC_ID_MPEG4      &&
461         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
462         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
463         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
464         return -1;
465     }
466
467     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
468          s->codec_id == AV_CODEC_ID_H263  ||
469          s->codec_id == AV_CODEC_ID_H263P) &&
470         (avctx->sample_aspect_ratio.num > 255 ||
471          avctx->sample_aspect_ratio.den > 255)) {
472         av_log(avctx, AV_LOG_ERROR,
473                "Invalid pixel aspect ratio %i/%i, limit is 255/255\n",
474                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
475         return -1;
476     }
477
478     if ((s->flags & (CODEC_FLAG_INTERLACED_DCT | CODEC_FLAG_INTERLACED_ME)) &&
479         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
480         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
481         return -1;
482     }
483
484     // FIXME mpeg2 uses that too
485     if (s->mpeg_quant && s->codec_id != AV_CODEC_ID_MPEG4) {
486         av_log(avctx, AV_LOG_ERROR,
487                "mpeg2 style quantization not supported by codec\n");
488         return -1;
489     }
490
491 #if FF_API_MPV_GLOBAL_OPTS
492     if (s->flags & CODEC_FLAG_CBP_RD)
493         s->mpv_flags |= FF_MPV_FLAG_CBP_RD;
494 #endif
495
496     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
497         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
498         return -1;
499     }
500
501     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
502         s->avctx->mb_decision != FF_MB_DECISION_RD) {
503         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
504         return -1;
505     }
506
507     if (s->avctx->scenechange_threshold < 1000000000 &&
508         (s->flags & CODEC_FLAG_CLOSED_GOP)) {
509         av_log(avctx, AV_LOG_ERROR,
510                "closed gop with scene change detection are not supported yet, "
511                "set threshold to 1000000000\n");
512         return -1;
513     }
514
515     if (s->flags & CODEC_FLAG_LOW_DELAY) {
516         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
517             av_log(avctx, AV_LOG_ERROR,
518                   "low delay forcing is only available for mpeg2\n");
519             return -1;
520         }
521         if (s->max_b_frames != 0) {
522             av_log(avctx, AV_LOG_ERROR,
523                    "b frames cannot be used with low delay\n");
524             return -1;
525         }
526     }
527
528     if (s->q_scale_type == 1) {
529         if (avctx->qmax > 12) {
530             av_log(avctx, AV_LOG_ERROR,
531                    "non linear quant only supports qmax <= 12 currently\n");
532             return -1;
533         }
534     }
535
536     if (s->avctx->thread_count > 1         &&
537         s->codec_id != AV_CODEC_ID_MPEG4      &&
538         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
539         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
540         (s->codec_id != AV_CODEC_ID_H263P)) {
541         av_log(avctx, AV_LOG_ERROR,
542                "multi threaded encoding not supported by codec\n");
543         return -1;
544     }
545
546     if (s->avctx->thread_count < 1) {
547         av_log(avctx, AV_LOG_ERROR,
548                "automatic thread number detection not supported by codec,"
549                "patch welcome\n");
550         return -1;
551     }
552
553     if (s->avctx->thread_count > 1)
554         s->rtp_mode = 1;
555
556     if (!avctx->time_base.den || !avctx->time_base.num) {
557         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
558         return -1;
559     }
560
561     i = (INT_MAX / 2 + 128) >> 8;
562     if (avctx->me_threshold >= i) {
563         av_log(avctx, AV_LOG_ERROR, "me_threshold too large, max is %d\n",
564                i - 1);
565         return -1;
566     }
567     if (avctx->mb_threshold >= i) {
568         av_log(avctx, AV_LOG_ERROR, "mb_threshold too large, max is %d\n",
569                i - 1);
570         return -1;
571     }
572
573     if (avctx->b_frame_strategy && (avctx->flags & CODEC_FLAG_PASS2)) {
574         av_log(avctx, AV_LOG_INFO,
575                "notice: b_frame_strategy only affects the first pass\n");
576         avctx->b_frame_strategy = 0;
577     }
578
579     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
580     if (i > 1) {
581         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
582         avctx->time_base.den /= i;
583         avctx->time_base.num /= i;
584         //return -1;
585     }
586
587     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
588         s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG) {
589         // (a + x * 3 / 8) / x
590         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
591         s->inter_quant_bias = 0;
592     } else {
593         s->intra_quant_bias = 0;
594         // (a - x / 4) / x
595         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
596     }
597
598     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
599         s->intra_quant_bias = avctx->intra_quant_bias;
600     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
601         s->inter_quant_bias = avctx->inter_quant_bias;
602
603     avcodec_get_chroma_sub_sample(avctx->pix_fmt, &chroma_h_shift,
604                                   &chroma_v_shift);
605
606     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
607         s->avctx->time_base.den > (1 << 16) - 1) {
608         av_log(avctx, AV_LOG_ERROR,
609                "timebase %d/%d not supported by MPEG 4 standard, "
610                "the maximum admitted value for the timebase denominator "
611                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
612                (1 << 16) - 1);
613         return -1;
614     }
615     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
616
617 #if FF_API_MPV_GLOBAL_OPTS
618     if (avctx->flags2 & CODEC_FLAG2_SKIP_RD)
619         s->mpv_flags |= FF_MPV_FLAG_SKIP_RD;
620     if (avctx->flags2 & CODEC_FLAG2_STRICT_GOP)
621         s->mpv_flags |= FF_MPV_FLAG_STRICT_GOP;
622     if (avctx->quantizer_noise_shaping)
623         s->quantizer_noise_shaping = avctx->quantizer_noise_shaping;
624 #endif
625
626     switch (avctx->codec->id) {
627     case AV_CODEC_ID_MPEG1VIDEO:
628         s->out_format = FMT_MPEG1;
629         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
630         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
631         break;
632     case AV_CODEC_ID_MPEG2VIDEO:
633         s->out_format = FMT_MPEG1;
634         s->low_delay  = !!(s->flags & CODEC_FLAG_LOW_DELAY);
635         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
636         s->rtp_mode   = 1;
637         break;
638     case AV_CODEC_ID_LJPEG:
639     case AV_CODEC_ID_MJPEG:
640         s->out_format = FMT_MJPEG;
641         s->intra_only = 1; /* force intra only for jpeg */
642         if (avctx->codec->id == AV_CODEC_ID_LJPEG &&
643             avctx->pix_fmt   == PIX_FMT_BGRA) {
644             s->mjpeg_vsample[0] = s->mjpeg_hsample[0] =
645             s->mjpeg_vsample[1] = s->mjpeg_hsample[1] =
646             s->mjpeg_vsample[2] = s->mjpeg_hsample[2] = 1;
647         } else {
648             s->mjpeg_vsample[0] = 2;
649             s->mjpeg_vsample[1] = 2 >> chroma_v_shift;
650             s->mjpeg_vsample[2] = 2 >> chroma_v_shift;
651             s->mjpeg_hsample[0] = 2;
652             s->mjpeg_hsample[1] = 2 >> chroma_h_shift;
653             s->mjpeg_hsample[2] = 2 >> chroma_h_shift;
654         }
655         if (!(CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER) ||
656             ff_mjpeg_encode_init(s) < 0)
657             return -1;
658         avctx->delay = 0;
659         s->low_delay = 1;
660         break;
661     case AV_CODEC_ID_H261:
662         if (!CONFIG_H261_ENCODER)
663             return -1;
664         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
665             av_log(avctx, AV_LOG_ERROR,
666                    "The specified picture size of %dx%d is not valid for the "
667                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
668                     s->width, s->height);
669             return -1;
670         }
671         s->out_format = FMT_H261;
672         avctx->delay  = 0;
673         s->low_delay  = 1;
674         break;
675     case AV_CODEC_ID_H263:
676         if (!CONFIG_H263_ENCODER)
677         return -1;
678         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
679                              s->width, s->height) == 8) {
680             av_log(avctx, AV_LOG_INFO,
681                    "The specified picture size of %dx%d is not valid for "
682                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
683                    "352x288, 704x576, and 1408x1152."
684                    "Try H.263+.\n", s->width, s->height);
685             return -1;
686         }
687         s->out_format = FMT_H263;
688         avctx->delay  = 0;
689         s->low_delay  = 1;
690         break;
691     case AV_CODEC_ID_H263P:
692         s->out_format = FMT_H263;
693         s->h263_plus  = 1;
694         /* Fx */
695         s->h263_aic        = (avctx->flags & CODEC_FLAG_AC_PRED) ? 1 : 0;
696         s->modified_quant  = s->h263_aic;
697         s->loop_filter     = (avctx->flags & CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
698         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
699
700         /* /Fx */
701         /* These are just to be sure */
702         avctx->delay = 0;
703         s->low_delay = 1;
704         break;
705     case AV_CODEC_ID_FLV1:
706         s->out_format      = FMT_H263;
707         s->h263_flv        = 2; /* format = 1; 11-bit codes */
708         s->unrestricted_mv = 1;
709         s->rtp_mode  = 0; /* don't allow GOB */
710         avctx->delay = 0;
711         s->low_delay = 1;
712         break;
713     case AV_CODEC_ID_RV10:
714         s->out_format = FMT_H263;
715         avctx->delay  = 0;
716         s->low_delay  = 1;
717         break;
718     case AV_CODEC_ID_RV20:
719         s->out_format      = FMT_H263;
720         avctx->delay       = 0;
721         s->low_delay       = 1;
722         s->modified_quant  = 1;
723         s->h263_aic        = 1;
724         s->h263_plus       = 1;
725         s->loop_filter     = 1;
726         s->unrestricted_mv = 0;
727         break;
728     case AV_CODEC_ID_MPEG4:
729         s->out_format      = FMT_H263;
730         s->h263_pred       = 1;
731         s->unrestricted_mv = 1;
732         s->low_delay       = s->max_b_frames ? 0 : 1;
733         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
734         break;
735     case AV_CODEC_ID_MSMPEG4V2:
736         s->out_format      = FMT_H263;
737         s->h263_pred       = 1;
738         s->unrestricted_mv = 1;
739         s->msmpeg4_version = 2;
740         avctx->delay       = 0;
741         s->low_delay       = 1;
742         break;
743     case AV_CODEC_ID_MSMPEG4V3:
744         s->out_format        = FMT_H263;
745         s->h263_pred         = 1;
746         s->unrestricted_mv   = 1;
747         s->msmpeg4_version   = 3;
748         s->flipflop_rounding = 1;
749         avctx->delay         = 0;
750         s->low_delay         = 1;
751         break;
752     case AV_CODEC_ID_WMV1:
753         s->out_format        = FMT_H263;
754         s->h263_pred         = 1;
755         s->unrestricted_mv   = 1;
756         s->msmpeg4_version   = 4;
757         s->flipflop_rounding = 1;
758         avctx->delay         = 0;
759         s->low_delay         = 1;
760         break;
761     case AV_CODEC_ID_WMV2:
762         s->out_format        = FMT_H263;
763         s->h263_pred         = 1;
764         s->unrestricted_mv   = 1;
765         s->msmpeg4_version   = 5;
766         s->flipflop_rounding = 1;
767         avctx->delay         = 0;
768         s->low_delay         = 1;
769         break;
770     default:
771         return -1;
772     }
773
774     avctx->has_b_frames = !s->low_delay;
775
776     s->encoding = 1;
777
778     s->progressive_frame    =
779     s->progressive_sequence = !(avctx->flags & (CODEC_FLAG_INTERLACED_DCT |
780                                                 CODEC_FLAG_INTERLACED_ME) ||
781                                 s->alternate_scan);
782
783     /* init */
784     if (ff_MPV_common_init(s) < 0)
785         return -1;
786
787     if (!s->dct_quantize)
788         s->dct_quantize = ff_dct_quantize_c;
789     if (!s->denoise_dct)
790         s->denoise_dct  = denoise_dct_c;
791     s->fast_dct_quantize = s->dct_quantize;
792     if (avctx->trellis)
793         s->dct_quantize  = dct_quantize_trellis_c;
794
795     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
796         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
797
798     s->quant_precision = 5;
799
800     ff_set_cmp(&s->dsp, s->dsp.ildct_cmp, s->avctx->ildct_cmp);
801     ff_set_cmp(&s->dsp, s->dsp.frame_skip_cmp, s->avctx->frame_skip_cmp);
802
803     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
804         ff_h261_encode_init(s);
805     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
806         ff_h263_encode_init(s);
807     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
808         ff_msmpeg4_encode_init(s);
809     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
810         && s->out_format == FMT_MPEG1)
811         ff_mpeg1_encode_init(s);
812
813     /* init q matrix */
814     for (i = 0; i < 64; i++) {
815         int j = s->dsp.idct_permutation[i];
816         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
817             s->mpeg_quant) {
818             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
819             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
820         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
821             s->intra_matrix[j] =
822             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
823         } else {
824             /* mpeg1/2 */
825             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
826             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
827         }
828         if (s->avctx->intra_matrix)
829             s->intra_matrix[j] = s->avctx->intra_matrix[i];
830         if (s->avctx->inter_matrix)
831             s->inter_matrix[j] = s->avctx->inter_matrix[i];
832     }
833
834     /* precompute matrix */
835     /* for mjpeg, we do include qscale in the matrix */
836     if (s->out_format != FMT_MJPEG) {
837         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
838                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
839                           31, 1);
840         ff_convert_matrix(&s->dsp, s->q_inter_matrix, s->q_inter_matrix16,
841                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
842                           31, 0);
843     }
844
845     if (ff_rate_control_init(s) < 0)
846         return -1;
847
848     return 0;
849 }
850
851 av_cold int ff_MPV_encode_end(AVCodecContext *avctx)
852 {
853     MpegEncContext *s = avctx->priv_data;
854
855     ff_rate_control_uninit(s);
856
857     ff_MPV_common_end(s);
858     if ((CONFIG_MJPEG_ENCODER || CONFIG_LJPEG_ENCODER) &&
859         s->out_format == FMT_MJPEG)
860         ff_mjpeg_encode_close(s);
861
862     av_freep(&avctx->extradata);
863
864     return 0;
865 }
866
867 static int get_sae(uint8_t *src, int ref, int stride)
868 {
869     int x,y;
870     int acc = 0;
871
872     for (y = 0; y < 16; y++) {
873         for (x = 0; x < 16; x++) {
874             acc += FFABS(src[x + y * stride] - ref);
875         }
876     }
877
878     return acc;
879 }
880
881 static int get_intra_count(MpegEncContext *s, uint8_t *src,
882                            uint8_t *ref, int stride)
883 {
884     int x, y, w, h;
885     int acc = 0;
886
887     w = s->width  & ~15;
888     h = s->height & ~15;
889
890     for (y = 0; y < h; y += 16) {
891         for (x = 0; x < w; x += 16) {
892             int offset = x + y * stride;
893             int sad  = s->dsp.sad[0](NULL, src + offset, ref + offset, stride,
894                                      16);
895             int mean = (s->dsp.pix_sum(src + offset, stride) + 128) >> 8;
896             int sae  = get_sae(src + offset, mean, stride);
897
898             acc += sae + 500 < sad;
899         }
900     }
901     return acc;
902 }
903
904
905 static int load_input_picture(MpegEncContext *s, AVFrame *pic_arg)
906 {
907     AVFrame *pic = NULL;
908     int64_t pts;
909     int i;
910     const int encoding_delay = s->max_b_frames ? s->max_b_frames :
911                                                  (s->low_delay ? 0 : 1);
912     int direct = 1;
913
914     if (pic_arg) {
915         pts = pic_arg->pts;
916         pic_arg->display_picture_number = s->input_picture_number++;
917
918         if (pts != AV_NOPTS_VALUE) {
919             if (s->user_specified_pts != AV_NOPTS_VALUE) {
920                 int64_t time = pts;
921                 int64_t last = s->user_specified_pts;
922
923                 if (time <= last) {
924                     av_log(s->avctx, AV_LOG_ERROR,
925                            "Error, Invalid timestamp=%"PRId64", "
926                            "last=%"PRId64"\n", pts, s->user_specified_pts);
927                     return -1;
928                 }
929
930                 if (!s->low_delay && pic_arg->display_picture_number == 1)
931                     s->dts_delta = time - last;
932             }
933             s->user_specified_pts = pts;
934         } else {
935             if (s->user_specified_pts != AV_NOPTS_VALUE) {
936                 s->user_specified_pts =
937                 pts = s->user_specified_pts + 1;
938                 av_log(s->avctx, AV_LOG_INFO,
939                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
940                        pts);
941             } else {
942                 pts = pic_arg->display_picture_number;
943             }
944         }
945     }
946
947   if (pic_arg) {
948     if (encoding_delay && !(s->flags & CODEC_FLAG_INPUT_PRESERVED))
949         direct = 0;
950     if (pic_arg->linesize[0] != s->linesize)
951         direct = 0;
952     if (pic_arg->linesize[1] != s->uvlinesize)
953         direct = 0;
954     if (pic_arg->linesize[2] != s->uvlinesize)
955         direct = 0;
956
957     //av_log(AV_LOG_DEBUG, "%d %d %d %d\n",pic_arg->linesize[0],
958     //       pic_arg->linesize[1], s->linesize, s->uvlinesize);
959
960     if (direct) {
961         i = ff_find_unused_picture(s, 1);
962         if (i < 0)
963             return i;
964
965         pic = &s->picture[i].f;
966         pic->reference = 3;
967
968         for (i = 0; i < 4; i++) {
969             pic->data[i]     = pic_arg->data[i];
970             pic->linesize[i] = pic_arg->linesize[i];
971         }
972         if (ff_alloc_picture(s, (Picture *) pic, 1) < 0) {
973             return -1;
974         }
975     } else {
976         i = ff_find_unused_picture(s, 0);
977         if (i < 0)
978             return i;
979
980         pic = &s->picture[i].f;
981         pic->reference = 3;
982
983         if (ff_alloc_picture(s, (Picture *) pic, 0) < 0) {
984             return -1;
985         }
986
987         if (pic->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
988             pic->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
989             pic->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
990             // empty
991         } else {
992             int h_chroma_shift, v_chroma_shift;
993             avcodec_get_chroma_sub_sample(s->avctx->pix_fmt, &h_chroma_shift,
994                                           &v_chroma_shift);
995
996             for (i = 0; i < 3; i++) {
997                 int src_stride = pic_arg->linesize[i];
998                 int dst_stride = i ? s->uvlinesize : s->linesize;
999                 int h_shift = i ? h_chroma_shift : 0;
1000                 int v_shift = i ? v_chroma_shift : 0;
1001                 int w = s->width  >> h_shift;
1002                 int h = s->height >> v_shift;
1003                 uint8_t *src = pic_arg->data[i];
1004                 uint8_t *dst = pic->data[i];
1005
1006                 if (!s->avctx->rc_buffer_size)
1007                     dst += INPLACE_OFFSET;
1008
1009                 if (src_stride == dst_stride)
1010                     memcpy(dst, src, src_stride * h);
1011                 else {
1012                     while (h--) {
1013                         memcpy(dst, src, w);
1014                         dst += dst_stride;
1015                         src += src_stride;
1016                     }
1017                 }
1018             }
1019         }
1020     }
1021     copy_picture_attributes(s, pic, pic_arg);
1022     pic->pts = pts; // we set this here to avoid modifiying pic_arg
1023   }
1024
1025     /* shift buffer entries */
1026     for (i = 1; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1027         s->input_picture[i - 1] = s->input_picture[i];
1028
1029     s->input_picture[encoding_delay] = (Picture*) pic;
1030
1031     return 0;
1032 }
1033
1034 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1035 {
1036     int x, y, plane;
1037     int score = 0;
1038     int64_t score64 = 0;
1039
1040     for (plane = 0; plane < 3; plane++) {
1041         const int stride = p->f.linesize[plane];
1042         const int bw = plane ? 1 : 2;
1043         for (y = 0; y < s->mb_height * bw; y++) {
1044             for (x = 0; x < s->mb_width * bw; x++) {
1045                 int off = p->f.type == FF_BUFFER_TYPE_SHARED ? 0 : 16;
1046                 uint8_t *dptr = p->f.data[plane] + 8 * (x + y * stride) + off;
1047                 uint8_t *rptr = ref->f.data[plane] + 8 * (x + y * stride);
1048                 int v   = s->dsp.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1049
1050                 switch (s->avctx->frame_skip_exp) {
1051                 case 0: score    =  FFMAX(score, v);          break;
1052                 case 1: score   += FFABS(v);                  break;
1053                 case 2: score   += v * v;                     break;
1054                 case 3: score64 += FFABS(v * v * (int64_t)v); break;
1055                 case 4: score64 += v * v * (int64_t)(v * v);  break;
1056                 }
1057             }
1058         }
1059     }
1060
1061     if (score)
1062         score64 = score;
1063
1064     if (score64 < s->avctx->frame_skip_threshold)
1065         return 1;
1066     if (score64 < ((s->avctx->frame_skip_factor * (int64_t)s->lambda) >> 8))
1067         return 1;
1068     return 0;
1069 }
1070
1071 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1072 {
1073     AVPacket pkt = { 0 };
1074     int ret, got_output;
1075
1076     av_init_packet(&pkt);
1077     av_init_packet(&pkt);
1078     ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
1079     if (ret < 0)
1080         return ret;
1081
1082     ret = pkt.size;
1083     av_free_packet(&pkt);
1084     return ret;
1085 }
1086
1087 static int estimate_best_b_count(MpegEncContext *s)
1088 {
1089     AVCodec *codec    = avcodec_find_encoder(s->avctx->codec_id);
1090     AVCodecContext *c = avcodec_alloc_context3(NULL);
1091     AVFrame input[FF_MAX_B_FRAMES + 2];
1092     const int scale = s->avctx->brd_scale;
1093     int i, j, out_size, p_lambda, b_lambda, lambda2;
1094     int64_t best_rd  = INT64_MAX;
1095     int best_b_count = -1;
1096
1097     assert(scale >= 0 && scale <= 3);
1098
1099     //emms_c();
1100     //s->next_picture_ptr->quality;
1101     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1102     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1103     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1104     if (!b_lambda) // FIXME we should do this somewhere else
1105         b_lambda = p_lambda;
1106     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1107                FF_LAMBDA_SHIFT;
1108
1109     c->width        = s->width  >> scale;
1110     c->height       = s->height >> scale;
1111     c->flags        = CODEC_FLAG_QSCALE | CODEC_FLAG_PSNR |
1112                       CODEC_FLAG_INPUT_PRESERVED /*| CODEC_FLAG_EMU_EDGE*/;
1113     c->flags       |= s->avctx->flags & CODEC_FLAG_QPEL;
1114     c->mb_decision  = s->avctx->mb_decision;
1115     c->me_cmp       = s->avctx->me_cmp;
1116     c->mb_cmp       = s->avctx->mb_cmp;
1117     c->me_sub_cmp   = s->avctx->me_sub_cmp;
1118     c->pix_fmt      = PIX_FMT_YUV420P;
1119     c->time_base    = s->avctx->time_base;
1120     c->max_b_frames = s->max_b_frames;
1121
1122     if (avcodec_open2(c, codec, NULL) < 0)
1123         return -1;
1124
1125     for (i = 0; i < s->max_b_frames + 2; i++) {
1126         int ysize = c->width * c->height;
1127         int csize = (c->width / 2) * (c->height / 2);
1128         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1129                                                 s->next_picture_ptr;
1130
1131         avcodec_get_frame_defaults(&input[i]);
1132         input[i].data[0]     = av_malloc(ysize + 2 * csize);
1133         input[i].data[1]     = input[i].data[0] + ysize;
1134         input[i].data[2]     = input[i].data[1] + csize;
1135         input[i].linesize[0] = c->width;
1136         input[i].linesize[1] =
1137         input[i].linesize[2] = c->width / 2;
1138
1139         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1140             pre_input = *pre_input_ptr;
1141
1142             if (pre_input.f.type != FF_BUFFER_TYPE_SHARED && i) {
1143                 pre_input.f.data[0] += INPLACE_OFFSET;
1144                 pre_input.f.data[1] += INPLACE_OFFSET;
1145                 pre_input.f.data[2] += INPLACE_OFFSET;
1146             }
1147
1148             s->dsp.shrink[scale](input[i].data[0], input[i].linesize[0],
1149                                  pre_input.f.data[0], pre_input.f.linesize[0],
1150                                  c->width,      c->height);
1151             s->dsp.shrink[scale](input[i].data[1], input[i].linesize[1],
1152                                  pre_input.f.data[1], pre_input.f.linesize[1],
1153                                  c->width >> 1, c->height >> 1);
1154             s->dsp.shrink[scale](input[i].data[2], input[i].linesize[2],
1155                                  pre_input.f.data[2], pre_input.f.linesize[2],
1156                                  c->width >> 1, c->height >> 1);
1157         }
1158     }
1159
1160     for (j = 0; j < s->max_b_frames + 1; j++) {
1161         int64_t rd = 0;
1162
1163         if (!s->input_picture[j])
1164             break;
1165
1166         c->error[0] = c->error[1] = c->error[2] = 0;
1167
1168         input[0].pict_type = AV_PICTURE_TYPE_I;
1169         input[0].quality   = 1 * FF_QP2LAMBDA;
1170
1171         out_size = encode_frame(c, &input[0]);
1172
1173         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1174
1175         for (i = 0; i < s->max_b_frames + 1; i++) {
1176             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1177
1178             input[i + 1].pict_type = is_p ?
1179                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1180             input[i + 1].quality   = is_p ? p_lambda : b_lambda;
1181
1182             out_size = encode_frame(c, &input[i + 1]);
1183
1184             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1185         }
1186
1187         /* get the delayed frames */
1188         while (out_size) {
1189             out_size = encode_frame(c, NULL);
1190             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1191         }
1192
1193         rd += c->error[0] + c->error[1] + c->error[2];
1194
1195         if (rd < best_rd) {
1196             best_rd = rd;
1197             best_b_count = j;
1198         }
1199     }
1200
1201     avcodec_close(c);
1202     av_freep(&c);
1203
1204     for (i = 0; i < s->max_b_frames + 2; i++) {
1205         av_freep(&input[i].data[0]);
1206     }
1207
1208     return best_b_count;
1209 }
1210
1211 static int select_input_picture(MpegEncContext *s)
1212 {
1213     int i;
1214
1215     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1216         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1217     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1218
1219     /* set next picture type & ordering */
1220     if (s->reordered_input_picture[0] == NULL && s->input_picture[0]) {
1221         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1222             s->next_picture_ptr == NULL || s->intra_only) {
1223             s->reordered_input_picture[0] = s->input_picture[0];
1224             s->reordered_input_picture[0]->f.pict_type = AV_PICTURE_TYPE_I;
1225             s->reordered_input_picture[0]->f.coded_picture_number =
1226                 s->coded_picture_number++;
1227         } else {
1228             int b_frames;
1229
1230             if (s->avctx->frame_skip_threshold || s->avctx->frame_skip_factor) {
1231                 if (s->picture_in_gop_number < s->gop_size &&
1232                     skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1233                     // FIXME check that te gop check above is +-1 correct
1234                     //av_log(NULL, AV_LOG_DEBUG, "skip %p %"PRId64"\n",
1235                     //       s->input_picture[0]->f.data[0],
1236                     //       s->input_picture[0]->pts);
1237
1238                     if (s->input_picture[0]->f.type == FF_BUFFER_TYPE_SHARED) {
1239                         for (i = 0; i < 4; i++)
1240                             s->input_picture[0]->f.data[i] = NULL;
1241                         s->input_picture[0]->f.type = 0;
1242                     } else {
1243                         assert(s->input_picture[0]->f.type == FF_BUFFER_TYPE_USER ||
1244                                s->input_picture[0]->f.type == FF_BUFFER_TYPE_INTERNAL);
1245
1246                         s->avctx->release_buffer(s->avctx,
1247                                                  &s->input_picture[0]->f);
1248                     }
1249
1250                     emms_c();
1251                     ff_vbv_update(s, 0);
1252
1253                     goto no_output_pic;
1254                 }
1255             }
1256
1257             if (s->flags & CODEC_FLAG_PASS2) {
1258                 for (i = 0; i < s->max_b_frames + 1; i++) {
1259                     int pict_num = s->input_picture[0]->f.display_picture_number + i;
1260
1261                     if (pict_num >= s->rc_context.num_entries)
1262                         break;
1263                     if (!s->input_picture[i]) {
1264                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1265                         break;
1266                     }
1267
1268                     s->input_picture[i]->f.pict_type =
1269                         s->rc_context.entry[pict_num].new_pict_type;
1270                 }
1271             }
1272
1273             if (s->avctx->b_frame_strategy == 0) {
1274                 b_frames = s->max_b_frames;
1275                 while (b_frames && !s->input_picture[b_frames])
1276                     b_frames--;
1277             } else if (s->avctx->b_frame_strategy == 1) {
1278                 for (i = 1; i < s->max_b_frames + 1; i++) {
1279                     if (s->input_picture[i] &&
1280                         s->input_picture[i]->b_frame_score == 0) {
1281                         s->input_picture[i]->b_frame_score =
1282                             get_intra_count(s,
1283                                             s->input_picture[i    ]->f.data[0],
1284                                             s->input_picture[i - 1]->f.data[0],
1285                                             s->linesize) + 1;
1286                     }
1287                 }
1288                 for (i = 0; i < s->max_b_frames + 1; i++) {
1289                     if (s->input_picture[i] == NULL ||
1290                         s->input_picture[i]->b_frame_score - 1 >
1291                             s->mb_num / s->avctx->b_sensitivity)
1292                         break;
1293                 }
1294
1295                 b_frames = FFMAX(0, i - 1);
1296
1297                 /* reset scores */
1298                 for (i = 0; i < b_frames + 1; i++) {
1299                     s->input_picture[i]->b_frame_score = 0;
1300                 }
1301             } else if (s->avctx->b_frame_strategy == 2) {
1302                 b_frames = estimate_best_b_count(s);
1303             } else {
1304                 av_log(s->avctx, AV_LOG_ERROR, "illegal b frame strategy\n");
1305                 b_frames = 0;
1306             }
1307
1308             emms_c();
1309             //static int b_count = 0;
1310             //b_count += b_frames;
1311             //av_log(s->avctx, AV_LOG_DEBUG, "b_frames: %d\n", b_count);
1312
1313             for (i = b_frames - 1; i >= 0; i--) {
1314                 int type = s->input_picture[i]->f.pict_type;
1315                 if (type && type != AV_PICTURE_TYPE_B)
1316                     b_frames = i;
1317             }
1318             if (s->input_picture[b_frames]->f.pict_type == AV_PICTURE_TYPE_B &&
1319                 b_frames == s->max_b_frames) {
1320                 av_log(s->avctx, AV_LOG_ERROR,
1321                        "warning, too many b frames in a row\n");
1322             }
1323
1324             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1325                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1326                     s->gop_size > s->picture_in_gop_number) {
1327                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1328                 } else {
1329                     if (s->flags & CODEC_FLAG_CLOSED_GOP)
1330                         b_frames = 0;
1331                     s->input_picture[b_frames]->f.pict_type = AV_PICTURE_TYPE_I;
1332                 }
1333             }
1334
1335             if ((s->flags & CODEC_FLAG_CLOSED_GOP) && b_frames &&
1336                 s->input_picture[b_frames]->f.pict_type == AV_PICTURE_TYPE_I)
1337                 b_frames--;
1338
1339             s->reordered_input_picture[0] = s->input_picture[b_frames];
1340             if (s->reordered_input_picture[0]->f.pict_type != AV_PICTURE_TYPE_I)
1341                 s->reordered_input_picture[0]->f.pict_type = AV_PICTURE_TYPE_P;
1342             s->reordered_input_picture[0]->f.coded_picture_number =
1343                 s->coded_picture_number++;
1344             for (i = 0; i < b_frames; i++) {
1345                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1346                 s->reordered_input_picture[i + 1]->f.pict_type =
1347                     AV_PICTURE_TYPE_B;
1348                 s->reordered_input_picture[i + 1]->f.coded_picture_number =
1349                     s->coded_picture_number++;
1350             }
1351         }
1352     }
1353 no_output_pic:
1354     if (s->reordered_input_picture[0]) {
1355         s->reordered_input_picture[0]->f.reference =
1356            s->reordered_input_picture[0]->f.pict_type !=
1357                AV_PICTURE_TYPE_B ? 3 : 0;
1358
1359         ff_copy_picture(&s->new_picture, s->reordered_input_picture[0]);
1360
1361         if (s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_SHARED ||
1362             s->avctx->rc_buffer_size) {
1363             // input is a shared pix, so we can't modifiy it -> alloc a new
1364             // one & ensure that the shared one is reuseable
1365
1366             Picture *pic;
1367             int i = ff_find_unused_picture(s, 0);
1368             if (i < 0)
1369                 return i;
1370             pic = &s->picture[i];
1371
1372             pic->f.reference = s->reordered_input_picture[0]->f.reference;
1373             if (ff_alloc_picture(s, pic, 0) < 0) {
1374                 return -1;
1375             }
1376
1377             /* mark us unused / free shared pic */
1378             if (s->reordered_input_picture[0]->f.type == FF_BUFFER_TYPE_INTERNAL)
1379                 s->avctx->release_buffer(s->avctx,
1380                                          &s->reordered_input_picture[0]->f);
1381             for (i = 0; i < 4; i++)
1382                 s->reordered_input_picture[0]->f.data[i] = NULL;
1383             s->reordered_input_picture[0]->f.type = 0;
1384
1385             copy_picture_attributes(s, &pic->f,
1386                                     &s->reordered_input_picture[0]->f);
1387
1388             s->current_picture_ptr = pic;
1389         } else {
1390             // input is not a shared pix -> reuse buffer for current_pix
1391
1392             assert(s->reordered_input_picture[0]->f.type ==
1393                        FF_BUFFER_TYPE_USER ||
1394                    s->reordered_input_picture[0]->f.type ==
1395                        FF_BUFFER_TYPE_INTERNAL);
1396
1397             s->current_picture_ptr = s->reordered_input_picture[0];
1398             for (i = 0; i < 4; i++) {
1399                 s->new_picture.f.data[i] += INPLACE_OFFSET;
1400             }
1401         }
1402         ff_copy_picture(&s->current_picture, s->current_picture_ptr);
1403
1404         s->picture_number = s->new_picture.f.display_picture_number;
1405         //printf("dpn:%d\n", s->picture_number);
1406     } else {
1407         memset(&s->new_picture, 0, sizeof(Picture));
1408     }
1409     return 0;
1410 }
1411
1412 int ff_MPV_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1413                           const AVFrame *pic_arg, int *got_packet)
1414 {
1415     MpegEncContext *s = avctx->priv_data;
1416     int i, stuffing_count, ret;
1417     int context_count = s->slice_context_count;
1418
1419     s->picture_in_gop_number++;
1420
1421     if (load_input_picture(s, pic_arg) < 0)
1422         return -1;
1423
1424     if (select_input_picture(s) < 0) {
1425         return -1;
1426     }
1427
1428     /* output? */
1429     if (s->new_picture.f.data[0]) {
1430         if (!pkt->data &&
1431             (ret = ff_alloc_packet(pkt, s->mb_width*s->mb_height*MAX_MB_BYTES)) < 0)
1432             return ret;
1433         if (s->mb_info) {
1434             s->mb_info_ptr = av_packet_new_side_data(pkt,
1435                                  AV_PKT_DATA_H263_MB_INFO,
1436                                  s->mb_width*s->mb_height*12);
1437             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1438         }
1439
1440         for (i = 0; i < context_count; i++) {
1441             int start_y = s->thread_context[i]->start_mb_y;
1442             int   end_y = s->thread_context[i]->  end_mb_y;
1443             int h       = s->mb_height;
1444             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1445             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1446
1447             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1448         }
1449
1450         s->pict_type = s->new_picture.f.pict_type;
1451         //emms_c();
1452         //printf("qs:%f %f %d\n", s->new_picture.quality,
1453         //       s->current_picture.quality, s->qscale);
1454         ff_MPV_frame_start(s, avctx);
1455 vbv_retry:
1456         if (encode_picture(s, s->picture_number) < 0)
1457             return -1;
1458
1459         avctx->header_bits = s->header_bits;
1460         avctx->mv_bits     = s->mv_bits;
1461         avctx->misc_bits   = s->misc_bits;
1462         avctx->i_tex_bits  = s->i_tex_bits;
1463         avctx->p_tex_bits  = s->p_tex_bits;
1464         avctx->i_count     = s->i_count;
1465         // FIXME f/b_count in avctx
1466         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1467         avctx->skip_count  = s->skip_count;
1468
1469         ff_MPV_frame_end(s);
1470
1471         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1472             ff_mjpeg_encode_picture_trailer(s);
1473
1474         if (avctx->rc_buffer_size) {
1475             RateControlContext *rcc = &s->rc_context;
1476             int max_size = rcc->buffer_index * avctx->rc_max_available_vbv_use;
1477
1478             if (put_bits_count(&s->pb) > max_size &&
1479                 s->lambda < s->avctx->lmax) {
1480                 s->next_lambda = FFMAX(s->lambda + 1, s->lambda *
1481                                        (s->qscale + 1) / s->qscale);
1482                 if (s->adaptive_quant) {
1483                     int i;
1484                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1485                         s->lambda_table[i] =
1486                             FFMAX(s->lambda_table[i] + 1,
1487                                   s->lambda_table[i] * (s->qscale + 1) /
1488                                   s->qscale);
1489                 }
1490                 s->mb_skipped = 0;        // done in MPV_frame_start()
1491                 // done in encode_picture() so we must undo it
1492                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1493                     if (s->flipflop_rounding          ||
1494                         s->codec_id == AV_CODEC_ID_H263P ||
1495                         s->codec_id == AV_CODEC_ID_MPEG4)
1496                         s->no_rounding ^= 1;
1497                 }
1498                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1499                     s->time_base       = s->last_time_base;
1500                     s->last_non_b_time = s->time - s->pp_time;
1501                 }
1502                 //av_log(NULL, AV_LOG_ERROR, "R:%d ", s->next_lambda);
1503                 for (i = 0; i < context_count; i++) {
1504                     PutBitContext *pb = &s->thread_context[i]->pb;
1505                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1506                 }
1507                 goto vbv_retry;
1508             }
1509
1510             assert(s->avctx->rc_max_rate);
1511         }
1512
1513         if (s->flags & CODEC_FLAG_PASS1)
1514             ff_write_pass1_stats(s);
1515
1516         for (i = 0; i < 4; i++) {
1517             s->current_picture_ptr->f.error[i] = s->current_picture.f.error[i];
1518             avctx->error[i] += s->current_picture_ptr->f.error[i];
1519         }
1520
1521         if (s->flags & CODEC_FLAG_PASS1)
1522             assert(avctx->header_bits + avctx->mv_bits + avctx->misc_bits +
1523                    avctx->i_tex_bits + avctx->p_tex_bits ==
1524                        put_bits_count(&s->pb));
1525         flush_put_bits(&s->pb);
1526         s->frame_bits  = put_bits_count(&s->pb);
1527
1528         stuffing_count = ff_vbv_update(s, s->frame_bits);
1529         if (stuffing_count) {
1530             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1531                     stuffing_count + 50) {
1532                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1533                 return -1;
1534             }
1535
1536             switch (s->codec_id) {
1537             case AV_CODEC_ID_MPEG1VIDEO:
1538             case AV_CODEC_ID_MPEG2VIDEO:
1539                 while (stuffing_count--) {
1540                     put_bits(&s->pb, 8, 0);
1541                 }
1542             break;
1543             case AV_CODEC_ID_MPEG4:
1544                 put_bits(&s->pb, 16, 0);
1545                 put_bits(&s->pb, 16, 0x1C3);
1546                 stuffing_count -= 4;
1547                 while (stuffing_count--) {
1548                     put_bits(&s->pb, 8, 0xFF);
1549                 }
1550             break;
1551             default:
1552                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1553             }
1554             flush_put_bits(&s->pb);
1555             s->frame_bits  = put_bits_count(&s->pb);
1556         }
1557
1558         /* update mpeg1/2 vbv_delay for CBR */
1559         if (s->avctx->rc_max_rate                          &&
1560             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1561             s->out_format == FMT_MPEG1                     &&
1562             90000LL * (avctx->rc_buffer_size - 1) <=
1563                 s->avctx->rc_max_rate * 0xFFFFLL) {
1564             int vbv_delay, min_delay;
1565             double inbits  = s->avctx->rc_max_rate *
1566                              av_q2d(s->avctx->time_base);
1567             int    minbits = s->frame_bits - 8 *
1568                              (s->vbv_delay_ptr - s->pb.buf - 1);
1569             double bits    = s->rc_context.buffer_index + minbits - inbits;
1570
1571             if (bits < 0)
1572                 av_log(s->avctx, AV_LOG_ERROR,
1573                        "Internal error, negative bits\n");
1574
1575             assert(s->repeat_first_field == 0);
1576
1577             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
1578             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
1579                         s->avctx->rc_max_rate;
1580
1581             vbv_delay = FFMAX(vbv_delay, min_delay);
1582
1583             assert(vbv_delay < 0xFFFF);
1584
1585             s->vbv_delay_ptr[0] &= 0xF8;
1586             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
1587             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
1588             s->vbv_delay_ptr[2] &= 0x07;
1589             s->vbv_delay_ptr[2] |= vbv_delay << 3;
1590             avctx->vbv_delay     = vbv_delay * 300;
1591         }
1592         s->total_bits     += s->frame_bits;
1593         avctx->frame_bits  = s->frame_bits;
1594
1595         pkt->pts = s->current_picture.f.pts;
1596         if (!s->low_delay) {
1597             if (!s->current_picture.f.coded_picture_number)
1598                 pkt->dts = pkt->pts - s->dts_delta;
1599             else
1600                 pkt->dts = s->reordered_pts;
1601             s->reordered_pts = s->input_picture[0]->f.pts;
1602         } else
1603             pkt->dts = pkt->pts;
1604         if (s->current_picture.f.key_frame)
1605             pkt->flags |= AV_PKT_FLAG_KEY;
1606         if (s->mb_info)
1607             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
1608     } else {
1609         assert((put_bits_ptr(&s->pb) == s->pb.buf));
1610         s->frame_bits = 0;
1611     }
1612     assert((s->frame_bits & 7) == 0);
1613
1614     pkt->size = s->frame_bits / 8;
1615     *got_packet = !!pkt->size;
1616     return 0;
1617 }
1618
1619 static inline void dct_single_coeff_elimination(MpegEncContext *s,
1620                                                 int n, int threshold)
1621 {
1622     static const char tab[64] = {
1623         3, 2, 2, 1, 1, 1, 1, 1,
1624         1, 1, 1, 1, 1, 1, 1, 1,
1625         1, 1, 1, 1, 1, 1, 1, 1,
1626         0, 0, 0, 0, 0, 0, 0, 0,
1627         0, 0, 0, 0, 0, 0, 0, 0,
1628         0, 0, 0, 0, 0, 0, 0, 0,
1629         0, 0, 0, 0, 0, 0, 0, 0,
1630         0, 0, 0, 0, 0, 0, 0, 0
1631     };
1632     int score = 0;
1633     int run = 0;
1634     int i;
1635     DCTELEM *block = s->block[n];
1636     const int last_index = s->block_last_index[n];
1637     int skip_dc;
1638
1639     if (threshold < 0) {
1640         skip_dc = 0;
1641         threshold = -threshold;
1642     } else
1643         skip_dc = 1;
1644
1645     /* Are all we could set to zero already zero? */
1646     if (last_index <= skip_dc - 1)
1647         return;
1648
1649     for (i = 0; i <= last_index; i++) {
1650         const int j = s->intra_scantable.permutated[i];
1651         const int level = FFABS(block[j]);
1652         if (level == 1) {
1653             if (skip_dc && i == 0)
1654                 continue;
1655             score += tab[run];
1656             run = 0;
1657         } else if (level > 1) {
1658             return;
1659         } else {
1660             run++;
1661         }
1662     }
1663     if (score >= threshold)
1664         return;
1665     for (i = skip_dc; i <= last_index; i++) {
1666         const int j = s->intra_scantable.permutated[i];
1667         block[j] = 0;
1668     }
1669     if (block[0])
1670         s->block_last_index[n] = 0;
1671     else
1672         s->block_last_index[n] = -1;
1673 }
1674
1675 static inline void clip_coeffs(MpegEncContext *s, DCTELEM *block,
1676                                int last_index)
1677 {
1678     int i;
1679     const int maxlevel = s->max_qcoeff;
1680     const int minlevel = s->min_qcoeff;
1681     int overflow = 0;
1682
1683     if (s->mb_intra) {
1684         i = 1; // skip clipping of intra dc
1685     } else
1686         i = 0;
1687
1688     for (; i <= last_index; i++) {
1689         const int j = s->intra_scantable.permutated[i];
1690         int level = block[j];
1691
1692         if (level > maxlevel) {
1693             level = maxlevel;
1694             overflow++;
1695         } else if (level < minlevel) {
1696             level = minlevel;
1697             overflow++;
1698         }
1699
1700         block[j] = level;
1701     }
1702
1703     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1704         av_log(s->avctx, AV_LOG_INFO,
1705                "warning, clipping %d dct coefficients to %d..%d\n",
1706                overflow, minlevel, maxlevel);
1707 }
1708
1709 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
1710 {
1711     int x, y;
1712     // FIXME optimize
1713     for (y = 0; y < 8; y++) {
1714         for (x = 0; x < 8; x++) {
1715             int x2, y2;
1716             int sum = 0;
1717             int sqr = 0;
1718             int count = 0;
1719
1720             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
1721                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
1722                     int v = ptr[x2 + y2 * stride];
1723                     sum += v;
1724                     sqr += v * v;
1725                     count++;
1726                 }
1727             }
1728             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
1729         }
1730     }
1731 }
1732
1733 static av_always_inline void encode_mb_internal(MpegEncContext *s,
1734                                                 int motion_x, int motion_y,
1735                                                 int mb_block_height,
1736                                                 int mb_block_count)
1737 {
1738     int16_t weight[8][64];
1739     DCTELEM orig[8][64];
1740     const int mb_x = s->mb_x;
1741     const int mb_y = s->mb_y;
1742     int i;
1743     int skip_dct[8];
1744     int dct_offset = s->linesize * 8; // default for progressive frames
1745     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
1746     int wrap_y, wrap_c;
1747
1748     for (i = 0; i < mb_block_count; i++)
1749         skip_dct[i] = s->skipdct;
1750
1751     if (s->adaptive_quant) {
1752         const int last_qp = s->qscale;
1753         const int mb_xy = mb_x + mb_y * s->mb_stride;
1754
1755         s->lambda = s->lambda_table[mb_xy];
1756         update_qscale(s);
1757
1758         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
1759             s->qscale = s->current_picture_ptr->f.qscale_table[mb_xy];
1760             s->dquant = s->qscale - last_qp;
1761
1762             if (s->out_format == FMT_H263) {
1763                 s->dquant = av_clip(s->dquant, -2, 2);
1764
1765                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
1766                     if (!s->mb_intra) {
1767                         if (s->pict_type == AV_PICTURE_TYPE_B) {
1768                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
1769                                 s->dquant = 0;
1770                         }
1771                         if (s->mv_type == MV_TYPE_8X8)
1772                             s->dquant = 0;
1773                     }
1774                 }
1775             }
1776         }
1777         ff_set_qscale(s, last_qp + s->dquant);
1778     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
1779         ff_set_qscale(s, s->qscale + s->dquant);
1780
1781     wrap_y = s->linesize;
1782     wrap_c = s->uvlinesize;
1783     ptr_y  = s->new_picture.f.data[0] +
1784              (mb_y * 16 * wrap_y)              + mb_x * 16;
1785     ptr_cb = s->new_picture.f.data[1] +
1786              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1787     ptr_cr = s->new_picture.f.data[2] +
1788              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
1789
1790     if (mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) {
1791         uint8_t *ebuf = s->edge_emu_buffer + 32;
1792         s->dsp.emulated_edge_mc(ebuf, ptr_y, wrap_y, 16, 16, mb_x * 16,
1793                                 mb_y * 16, s->width, s->height);
1794         ptr_y = ebuf;
1795         s->dsp.emulated_edge_mc(ebuf + 18 * wrap_y, ptr_cb, wrap_c, 8,
1796                                 mb_block_height, mb_x * 8, mb_y * 8,
1797                                 s->width >> 1, s->height >> 1);
1798         ptr_cb = ebuf + 18 * wrap_y;
1799         s->dsp.emulated_edge_mc(ebuf + 18 * wrap_y + 8, ptr_cr, wrap_c, 8,
1800                                 mb_block_height, mb_x * 8, mb_y * 8,
1801                                 s->width >> 1, s->height >> 1);
1802         ptr_cr = ebuf + 18 * wrap_y + 8;
1803     }
1804
1805     if (s->mb_intra) {
1806         if (s->flags & CODEC_FLAG_INTERLACED_DCT) {
1807             int progressive_score, interlaced_score;
1808
1809             s->interlaced_dct = 0;
1810             progressive_score = s->dsp.ildct_cmp[4](s, ptr_y,
1811                                                     NULL, wrap_y, 8) +
1812                                 s->dsp.ildct_cmp[4](s, ptr_y + wrap_y * 8,
1813                                                     NULL, wrap_y, 8) - 400;
1814
1815             if (progressive_score > 0) {
1816                 interlaced_score = s->dsp.ildct_cmp[4](s, ptr_y,
1817                                                        NULL, wrap_y * 2, 8) +
1818                                    s->dsp.ildct_cmp[4](s, ptr_y + wrap_y,
1819                                                        NULL, wrap_y * 2, 8);
1820                 if (progressive_score > interlaced_score) {
1821                     s->interlaced_dct = 1;
1822
1823                     dct_offset = wrap_y;
1824                     wrap_y <<= 1;
1825                     if (s->chroma_format == CHROMA_422)
1826                         wrap_c <<= 1;
1827                 }
1828             }
1829         }
1830
1831         s->dsp.get_pixels(s->block[0], ptr_y                  , wrap_y);
1832         s->dsp.get_pixels(s->block[1], ptr_y              + 8 , wrap_y);
1833         s->dsp.get_pixels(s->block[2], ptr_y + dct_offset     , wrap_y);
1834         s->dsp.get_pixels(s->block[3], ptr_y + dct_offset + 8 , wrap_y);
1835
1836         if (s->flags & CODEC_FLAG_GRAY) {
1837             skip_dct[4] = 1;
1838             skip_dct[5] = 1;
1839         } else {
1840             s->dsp.get_pixels(s->block[4], ptr_cb, wrap_c);
1841             s->dsp.get_pixels(s->block[5], ptr_cr, wrap_c);
1842             if (!s->chroma_y_shift) { /* 422 */
1843                 s->dsp.get_pixels(s->block[6],
1844                                   ptr_cb + (dct_offset >> 1), wrap_c);
1845                 s->dsp.get_pixels(s->block[7],
1846                                   ptr_cr + (dct_offset >> 1), wrap_c);
1847             }
1848         }
1849     } else {
1850         op_pixels_func (*op_pix)[4];
1851         qpel_mc_func (*op_qpix)[16];
1852         uint8_t *dest_y, *dest_cb, *dest_cr;
1853
1854         dest_y  = s->dest[0];
1855         dest_cb = s->dest[1];
1856         dest_cr = s->dest[2];
1857
1858         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
1859             op_pix  = s->dsp.put_pixels_tab;
1860             op_qpix = s->dsp.put_qpel_pixels_tab;
1861         } else {
1862             op_pix  = s->dsp.put_no_rnd_pixels_tab;
1863             op_qpix = s->dsp.put_no_rnd_qpel_pixels_tab;
1864         }
1865
1866         if (s->mv_dir & MV_DIR_FORWARD) {
1867             ff_MPV_motion(s, dest_y, dest_cb, dest_cr, 0,
1868                           s->last_picture.f.data,
1869                           op_pix, op_qpix);
1870             op_pix  = s->dsp.avg_pixels_tab;
1871             op_qpix = s->dsp.avg_qpel_pixels_tab;
1872         }
1873         if (s->mv_dir & MV_DIR_BACKWARD) {
1874             ff_MPV_motion(s, dest_y, dest_cb, dest_cr, 1,
1875                           s->next_picture.f.data,
1876                           op_pix, op_qpix);
1877         }
1878
1879         if (s->flags & CODEC_FLAG_INTERLACED_DCT) {
1880             int progressive_score, interlaced_score;
1881
1882             s->interlaced_dct = 0;
1883             progressive_score = s->dsp.ildct_cmp[0](s, dest_y,
1884                                                     ptr_y,              wrap_y,
1885                                                     8) +
1886                                 s->dsp.ildct_cmp[0](s, dest_y + wrap_y * 8,
1887                                                     ptr_y + wrap_y * 8, wrap_y,
1888                                                     8) - 400;
1889
1890             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
1891                 progressive_score -= 400;
1892
1893             if (progressive_score > 0) {
1894                 interlaced_score = s->dsp.ildct_cmp[0](s, dest_y,
1895                                                        ptr_y,
1896                                                        wrap_y * 2, 8) +
1897                                    s->dsp.ildct_cmp[0](s, dest_y + wrap_y,
1898                                                        ptr_y + wrap_y,
1899                                                        wrap_y * 2, 8);
1900
1901                 if (progressive_score > interlaced_score) {
1902                     s->interlaced_dct = 1;
1903
1904                     dct_offset = wrap_y;
1905                     wrap_y <<= 1;
1906                     if (s->chroma_format == CHROMA_422)
1907                         wrap_c <<= 1;
1908                 }
1909             }
1910         }
1911
1912         s->dsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
1913         s->dsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
1914         s->dsp.diff_pixels(s->block[2], ptr_y + dct_offset,
1915                            dest_y + dct_offset, wrap_y);
1916         s->dsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
1917                            dest_y + dct_offset + 8, wrap_y);
1918
1919         if (s->flags & CODEC_FLAG_GRAY) {
1920             skip_dct[4] = 1;
1921             skip_dct[5] = 1;
1922         } else {
1923             s->dsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
1924             s->dsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
1925             if (!s->chroma_y_shift) { /* 422 */
1926                 s->dsp.diff_pixels(s->block[6], ptr_cb + (dct_offset >> 1),
1927                                    dest_cb + (dct_offset >> 1), wrap_c);
1928                 s->dsp.diff_pixels(s->block[7], ptr_cr + (dct_offset >> 1),
1929                                    dest_cr + (dct_offset >> 1), wrap_c);
1930             }
1931         }
1932         /* pre quantization */
1933         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
1934                 2 * s->qscale * s->qscale) {
1935             // FIXME optimize
1936             if (s->dsp.sad[1](NULL, ptr_y , dest_y,
1937                               wrap_y, 8) < 20 * s->qscale)
1938                 skip_dct[0] = 1;
1939             if (s->dsp.sad[1](NULL, ptr_y + 8,
1940                               dest_y + 8, wrap_y, 8) < 20 * s->qscale)
1941                 skip_dct[1] = 1;
1942             if (s->dsp.sad[1](NULL, ptr_y + dct_offset,
1943                               dest_y + dct_offset, wrap_y, 8) < 20 * s->qscale)
1944                 skip_dct[2] = 1;
1945             if (s->dsp.sad[1](NULL, ptr_y + dct_offset + 8,
1946                               dest_y + dct_offset + 8,
1947                               wrap_y, 8) < 20 * s->qscale)
1948                 skip_dct[3] = 1;
1949             if (s->dsp.sad[1](NULL, ptr_cb, dest_cb,
1950                               wrap_c, 8) < 20 * s->qscale)
1951                 skip_dct[4] = 1;
1952             if (s->dsp.sad[1](NULL, ptr_cr, dest_cr,
1953                               wrap_c, 8) < 20 * s->qscale)
1954                 skip_dct[5] = 1;
1955             if (!s->chroma_y_shift) { /* 422 */
1956                 if (s->dsp.sad[1](NULL, ptr_cb + (dct_offset >> 1),
1957                                   dest_cb + (dct_offset >> 1),
1958                                   wrap_c, 8) < 20 * s->qscale)
1959                     skip_dct[6] = 1;
1960                 if (s->dsp.sad[1](NULL, ptr_cr + (dct_offset >> 1),
1961                                   dest_cr + (dct_offset >> 1),
1962                                   wrap_c, 8) < 20 * s->qscale)
1963                     skip_dct[7] = 1;
1964             }
1965         }
1966     }
1967
1968     if (s->quantizer_noise_shaping) {
1969         if (!skip_dct[0])
1970             get_visual_weight(weight[0], ptr_y                 , wrap_y);
1971         if (!skip_dct[1])
1972             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
1973         if (!skip_dct[2])
1974             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
1975         if (!skip_dct[3])
1976             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
1977         if (!skip_dct[4])
1978             get_visual_weight(weight[4], ptr_cb                , wrap_c);
1979         if (!skip_dct[5])
1980             get_visual_weight(weight[5], ptr_cr                , wrap_c);
1981         if (!s->chroma_y_shift) { /* 422 */
1982             if (!skip_dct[6])
1983                 get_visual_weight(weight[6], ptr_cb + (dct_offset >> 1),
1984                                   wrap_c);
1985             if (!skip_dct[7])
1986                 get_visual_weight(weight[7], ptr_cr + (dct_offset >> 1),
1987                                   wrap_c);
1988         }
1989         memcpy(orig[0], s->block[0], sizeof(DCTELEM) * 64 * mb_block_count);
1990     }
1991
1992     /* DCT & quantize */
1993     assert(s->out_format != FMT_MJPEG || s->qscale == 8);
1994     {
1995         for (i = 0; i < mb_block_count; i++) {
1996             if (!skip_dct[i]) {
1997                 int overflow;
1998                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
1999                 // FIXME we could decide to change to quantizer instead of
2000                 // clipping
2001                 // JS: I don't think that would be a good idea it could lower
2002                 //     quality instead of improve it. Just INTRADC clipping
2003                 //     deserves changes in quantizer
2004                 if (overflow)
2005                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2006             } else
2007                 s->block_last_index[i] = -1;
2008         }
2009         if (s->quantizer_noise_shaping) {
2010             for (i = 0; i < mb_block_count; i++) {
2011                 if (!skip_dct[i]) {
2012                     s->block_last_index[i] =
2013                         dct_quantize_refine(s, s->block[i], weight[i],
2014                                             orig[i], i, s->qscale);
2015                 }
2016             }
2017         }
2018
2019         if (s->luma_elim_threshold && !s->mb_intra)
2020             for (i = 0; i < 4; i++)
2021                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2022         if (s->chroma_elim_threshold && !s->mb_intra)
2023             for (i = 4; i < mb_block_count; i++)
2024                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2025
2026         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2027             for (i = 0; i < mb_block_count; i++) {
2028                 if (s->block_last_index[i] == -1)
2029                     s->coded_score[i] = INT_MAX / 256;
2030             }
2031         }
2032     }
2033
2034     if ((s->flags & CODEC_FLAG_GRAY) && s->mb_intra) {
2035         s->block_last_index[4] =
2036         s->block_last_index[5] = 0;
2037         s->block[4][0] =
2038         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2039     }
2040
2041     // non c quantize code returns incorrect block_last_index FIXME
2042     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2043         for (i = 0; i < mb_block_count; i++) {
2044             int j;
2045             if (s->block_last_index[i] > 0) {
2046                 for (j = 63; j > 0; j--) {
2047                     if (s->block[i][s->intra_scantable.permutated[j]])
2048                         break;
2049                 }
2050                 s->block_last_index[i] = j;
2051             }
2052         }
2053     }
2054
2055     /* huffman encode */
2056     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2057     case AV_CODEC_ID_MPEG1VIDEO:
2058     case AV_CODEC_ID_MPEG2VIDEO:
2059         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2060             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2061         break;
2062     case AV_CODEC_ID_MPEG4:
2063         if (CONFIG_MPEG4_ENCODER)
2064             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2065         break;
2066     case AV_CODEC_ID_MSMPEG4V2:
2067     case AV_CODEC_ID_MSMPEG4V3:
2068     case AV_CODEC_ID_WMV1:
2069         if (CONFIG_MSMPEG4_ENCODER)
2070             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2071         break;
2072     case AV_CODEC_ID_WMV2:
2073         if (CONFIG_WMV2_ENCODER)
2074             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2075         break;
2076     case AV_CODEC_ID_H261:
2077         if (CONFIG_H261_ENCODER)
2078             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2079         break;
2080     case AV_CODEC_ID_H263:
2081     case AV_CODEC_ID_H263P:
2082     case AV_CODEC_ID_FLV1:
2083     case AV_CODEC_ID_RV10:
2084     case AV_CODEC_ID_RV20:
2085         if (CONFIG_H263_ENCODER)
2086             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2087         break;
2088     case AV_CODEC_ID_MJPEG:
2089         if (CONFIG_MJPEG_ENCODER)
2090             ff_mjpeg_encode_mb(s, s->block);
2091         break;
2092     default:
2093         assert(0);
2094     }
2095 }
2096
2097 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2098 {
2099     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
2100     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
2101 }
2102
2103 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2104     int i;
2105
2106     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2107
2108     /* mpeg1 */
2109     d->mb_skip_run= s->mb_skip_run;
2110     for(i=0; i<3; i++)
2111         d->last_dc[i] = s->last_dc[i];
2112
2113     /* statistics */
2114     d->mv_bits= s->mv_bits;
2115     d->i_tex_bits= s->i_tex_bits;
2116     d->p_tex_bits= s->p_tex_bits;
2117     d->i_count= s->i_count;
2118     d->f_count= s->f_count;
2119     d->b_count= s->b_count;
2120     d->skip_count= s->skip_count;
2121     d->misc_bits= s->misc_bits;
2122     d->last_bits= 0;
2123
2124     d->mb_skipped= 0;
2125     d->qscale= s->qscale;
2126     d->dquant= s->dquant;
2127
2128     d->esc3_level_length= s->esc3_level_length;
2129 }
2130
2131 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2132     int i;
2133
2134     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2135     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2136
2137     /* mpeg1 */
2138     d->mb_skip_run= s->mb_skip_run;
2139     for(i=0; i<3; i++)
2140         d->last_dc[i] = s->last_dc[i];
2141
2142     /* statistics */
2143     d->mv_bits= s->mv_bits;
2144     d->i_tex_bits= s->i_tex_bits;
2145     d->p_tex_bits= s->p_tex_bits;
2146     d->i_count= s->i_count;
2147     d->f_count= s->f_count;
2148     d->b_count= s->b_count;
2149     d->skip_count= s->skip_count;
2150     d->misc_bits= s->misc_bits;
2151
2152     d->mb_intra= s->mb_intra;
2153     d->mb_skipped= s->mb_skipped;
2154     d->mv_type= s->mv_type;
2155     d->mv_dir= s->mv_dir;
2156     d->pb= s->pb;
2157     if(s->data_partitioning){
2158         d->pb2= s->pb2;
2159         d->tex_pb= s->tex_pb;
2160     }
2161     d->block= s->block;
2162     for(i=0; i<8; i++)
2163         d->block_last_index[i]= s->block_last_index[i];
2164     d->interlaced_dct= s->interlaced_dct;
2165     d->qscale= s->qscale;
2166
2167     d->esc3_level_length= s->esc3_level_length;
2168 }
2169
2170 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2171                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2172                            int *dmin, int *next_block, int motion_x, int motion_y)
2173 {
2174     int score;
2175     uint8_t *dest_backup[3];
2176
2177     copy_context_before_encode(s, backup, type);
2178
2179     s->block= s->blocks[*next_block];
2180     s->pb= pb[*next_block];
2181     if(s->data_partitioning){
2182         s->pb2   = pb2   [*next_block];
2183         s->tex_pb= tex_pb[*next_block];
2184     }
2185
2186     if(*next_block){
2187         memcpy(dest_backup, s->dest, sizeof(s->dest));
2188         s->dest[0] = s->rd_scratchpad;
2189         s->dest[1] = s->rd_scratchpad + 16*s->linesize;
2190         s->dest[2] = s->rd_scratchpad + 16*s->linesize + 8;
2191         assert(s->linesize >= 32); //FIXME
2192     }
2193
2194     encode_mb(s, motion_x, motion_y);
2195
2196     score= put_bits_count(&s->pb);
2197     if(s->data_partitioning){
2198         score+= put_bits_count(&s->pb2);
2199         score+= put_bits_count(&s->tex_pb);
2200     }
2201
2202     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2203         ff_MPV_decode_mb(s, s->block);
2204
2205         score *= s->lambda2;
2206         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2207     }
2208
2209     if(*next_block){
2210         memcpy(s->dest, dest_backup, sizeof(s->dest));
2211     }
2212
2213     if(score<*dmin){
2214         *dmin= score;
2215         *next_block^=1;
2216
2217         copy_context_after_encode(best, s, type);
2218     }
2219 }
2220
2221 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2222     uint32_t *sq = ff_squareTbl + 256;
2223     int acc=0;
2224     int x,y;
2225
2226     if(w==16 && h==16)
2227         return s->dsp.sse[0](NULL, src1, src2, stride, 16);
2228     else if(w==8 && h==8)
2229         return s->dsp.sse[1](NULL, src1, src2, stride, 8);
2230
2231     for(y=0; y<h; y++){
2232         for(x=0; x<w; x++){
2233             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2234         }
2235     }
2236
2237     assert(acc>=0);
2238
2239     return acc;
2240 }
2241
2242 static int sse_mb(MpegEncContext *s){
2243     int w= 16;
2244     int h= 16;
2245
2246     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2247     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2248
2249     if(w==16 && h==16)
2250       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2251         return  s->dsp.nsse[0](s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2252                +s->dsp.nsse[1](s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2253                +s->dsp.nsse[1](s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2254       }else{
2255         return  s->dsp.sse[0](NULL, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], s->linesize, 16)
2256                +s->dsp.sse[1](NULL, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], s->uvlinesize, 8)
2257                +s->dsp.sse[1](NULL, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], s->uvlinesize, 8);
2258       }
2259     else
2260         return  sse(s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2261                +sse(s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2262                +sse(s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2263 }
2264
2265 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2266     MpegEncContext *s= *(void**)arg;
2267
2268
2269     s->me.pre_pass=1;
2270     s->me.dia_size= s->avctx->pre_dia_size;
2271     s->first_slice_line=1;
2272     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2273         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2274             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2275         }
2276         s->first_slice_line=0;
2277     }
2278
2279     s->me.pre_pass=0;
2280
2281     return 0;
2282 }
2283
2284 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2285     MpegEncContext *s= *(void**)arg;
2286
2287     ff_check_alignment();
2288
2289     s->me.dia_size= s->avctx->dia_size;
2290     s->first_slice_line=1;
2291     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2292         s->mb_x=0; //for block init below
2293         ff_init_block_index(s);
2294         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2295             s->block_index[0]+=2;
2296             s->block_index[1]+=2;
2297             s->block_index[2]+=2;
2298             s->block_index[3]+=2;
2299
2300             /* compute motion vector & mb_type and store in context */
2301             if(s->pict_type==AV_PICTURE_TYPE_B)
2302                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2303             else
2304                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2305         }
2306         s->first_slice_line=0;
2307     }
2308     return 0;
2309 }
2310
2311 static int mb_var_thread(AVCodecContext *c, void *arg){
2312     MpegEncContext *s= *(void**)arg;
2313     int mb_x, mb_y;
2314
2315     ff_check_alignment();
2316
2317     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2318         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2319             int xx = mb_x * 16;
2320             int yy = mb_y * 16;
2321             uint8_t *pix = s->new_picture.f.data[0] + (yy * s->linesize) + xx;
2322             int varc;
2323             int sum = s->dsp.pix_sum(pix, s->linesize);
2324
2325             varc = (s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)sum*sum)>>8) + 500 + 128)>>8;
2326
2327             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2328             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2329             s->me.mb_var_sum_temp    += varc;
2330         }
2331     }
2332     return 0;
2333 }
2334
2335 static void write_slice_end(MpegEncContext *s){
2336     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2337         if(s->partitioned_frame){
2338             ff_mpeg4_merge_partitions(s);
2339         }
2340
2341         ff_mpeg4_stuffing(&s->pb);
2342     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2343         ff_mjpeg_encode_stuffing(&s->pb);
2344     }
2345
2346     avpriv_align_put_bits(&s->pb);
2347     flush_put_bits(&s->pb);
2348
2349     if((s->flags&CODEC_FLAG_PASS1) && !s->partitioned_frame)
2350         s->misc_bits+= get_bits_diff(s);
2351 }
2352
2353 static void write_mb_info(MpegEncContext *s)
2354 {
2355     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2356     int offset = put_bits_count(&s->pb);
2357     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2358     int gobn = s->mb_y / s->gob_index;
2359     int pred_x, pred_y;
2360     if (CONFIG_H263_ENCODER)
2361         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2362     bytestream_put_le32(&ptr, offset);
2363     bytestream_put_byte(&ptr, s->qscale);
2364     bytestream_put_byte(&ptr, gobn);
2365     bytestream_put_le16(&ptr, mba);
2366     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2367     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2368     /* 4MV not implemented */
2369     bytestream_put_byte(&ptr, 0); /* hmv2 */
2370     bytestream_put_byte(&ptr, 0); /* vmv2 */
2371 }
2372
2373 static void update_mb_info(MpegEncContext *s, int startcode)
2374 {
2375     if (!s->mb_info)
2376         return;
2377     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2378         s->mb_info_size += 12;
2379         s->prev_mb_info = s->last_mb_info;
2380     }
2381     if (startcode) {
2382         s->prev_mb_info = put_bits_count(&s->pb)/8;
2383         /* This might have incremented mb_info_size above, and we return without
2384          * actually writing any info into that slot yet. But in that case,
2385          * this will be called again at the start of the after writing the
2386          * start code, actually writing the mb info. */
2387         return;
2388     }
2389
2390     s->last_mb_info = put_bits_count(&s->pb)/8;
2391     if (!s->mb_info_size)
2392         s->mb_info_size += 12;
2393     write_mb_info(s);
2394 }
2395
2396 static int encode_thread(AVCodecContext *c, void *arg){
2397     MpegEncContext *s= *(void**)arg;
2398     int mb_x, mb_y, pdif = 0;
2399     int chr_h= 16>>s->chroma_y_shift;
2400     int i, j;
2401     MpegEncContext best_s, backup_s;
2402     uint8_t bit_buf[2][MAX_MB_BYTES];
2403     uint8_t bit_buf2[2][MAX_MB_BYTES];
2404     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2405     PutBitContext pb[2], pb2[2], tex_pb[2];
2406 //printf("%d->%d\n", s->resync_mb_y, s->end_mb_y);
2407
2408     ff_check_alignment();
2409
2410     for(i=0; i<2; i++){
2411         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2412         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2413         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2414     }
2415
2416     s->last_bits= put_bits_count(&s->pb);
2417     s->mv_bits=0;
2418     s->misc_bits=0;
2419     s->i_tex_bits=0;
2420     s->p_tex_bits=0;
2421     s->i_count=0;
2422     s->f_count=0;
2423     s->b_count=0;
2424     s->skip_count=0;
2425
2426     for(i=0; i<3; i++){
2427         /* init last dc values */
2428         /* note: quant matrix value (8) is implied here */
2429         s->last_dc[i] = 128 << s->intra_dc_precision;
2430
2431         s->current_picture.f.error[i] = 0;
2432     }
2433     s->mb_skip_run = 0;
2434     memset(s->last_mv, 0, sizeof(s->last_mv));
2435
2436     s->last_mv_dir = 0;
2437
2438     switch(s->codec_id){
2439     case AV_CODEC_ID_H263:
2440     case AV_CODEC_ID_H263P:
2441     case AV_CODEC_ID_FLV1:
2442         if (CONFIG_H263_ENCODER)
2443             s->gob_index = ff_h263_get_gob_height(s);
2444         break;
2445     case AV_CODEC_ID_MPEG4:
2446         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2447             ff_mpeg4_init_partitions(s);
2448         break;
2449     }
2450
2451     s->resync_mb_x=0;
2452     s->resync_mb_y=0;
2453     s->first_slice_line = 1;
2454     s->ptr_lastgob = s->pb.buf;
2455     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2456 //    printf("row %d at %X\n", s->mb_y, (int)s);
2457         s->mb_x=0;
2458         s->mb_y= mb_y;
2459
2460         ff_set_qscale(s, s->qscale);
2461         ff_init_block_index(s);
2462
2463         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2464             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2465             int mb_type= s->mb_type[xy];
2466 //            int d;
2467             int dmin= INT_MAX;
2468             int dir;
2469
2470             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2471                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2472                 return -1;
2473             }
2474             if(s->data_partitioning){
2475                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2476                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2477                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2478                     return -1;
2479                 }
2480             }
2481
2482             s->mb_x = mb_x;
2483             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2484             ff_update_block_index(s);
2485
2486             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
2487                 ff_h261_reorder_mb_index(s);
2488                 xy= s->mb_y*s->mb_stride + s->mb_x;
2489                 mb_type= s->mb_type[xy];
2490             }
2491
2492             /* write gob / video packet header  */
2493             if(s->rtp_mode){
2494                 int current_packet_size, is_gob_start;
2495
2496                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2497
2498                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2499
2500                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2501
2502                 switch(s->codec_id){
2503                 case AV_CODEC_ID_H263:
2504                 case AV_CODEC_ID_H263P:
2505                     if(!s->h263_slice_structured)
2506                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2507                     break;
2508                 case AV_CODEC_ID_MPEG2VIDEO:
2509                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2510                 case AV_CODEC_ID_MPEG1VIDEO:
2511                     if(s->mb_skip_run) is_gob_start=0;
2512                     break;
2513                 }
2514
2515                 if(is_gob_start){
2516                     if(s->start_mb_y != mb_y || mb_x!=0){
2517                         write_slice_end(s);
2518
2519                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
2520                             ff_mpeg4_init_partitions(s);
2521                         }
2522                     }
2523
2524                     assert((put_bits_count(&s->pb)&7) == 0);
2525                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2526
2527                     if(s->avctx->error_rate && s->resync_mb_x + s->resync_mb_y > 0){
2528                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2529                         int d= 100 / s->avctx->error_rate;
2530                         if(r % d == 0){
2531                             current_packet_size=0;
2532                             s->pb.buf_ptr= s->ptr_lastgob;
2533                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2534                         }
2535                     }
2536
2537                     if (s->avctx->rtp_callback){
2538                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2539                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2540                     }
2541                     update_mb_info(s, 1);
2542
2543                     switch(s->codec_id){
2544                     case AV_CODEC_ID_MPEG4:
2545                         if (CONFIG_MPEG4_ENCODER) {
2546                             ff_mpeg4_encode_video_packet_header(s);
2547                             ff_mpeg4_clean_buffers(s);
2548                         }
2549                     break;
2550                     case AV_CODEC_ID_MPEG1VIDEO:
2551                     case AV_CODEC_ID_MPEG2VIDEO:
2552                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2553                             ff_mpeg1_encode_slice_header(s);
2554                             ff_mpeg1_clean_buffers(s);
2555                         }
2556                     break;
2557                     case AV_CODEC_ID_H263:
2558                     case AV_CODEC_ID_H263P:
2559                         if (CONFIG_H263_ENCODER)
2560                             ff_h263_encode_gob_header(s, mb_y);
2561                     break;
2562                     }
2563
2564                     if(s->flags&CODEC_FLAG_PASS1){
2565                         int bits= put_bits_count(&s->pb);
2566                         s->misc_bits+= bits - s->last_bits;
2567                         s->last_bits= bits;
2568                     }
2569
2570                     s->ptr_lastgob += current_packet_size;
2571                     s->first_slice_line=1;
2572                     s->resync_mb_x=mb_x;
2573                     s->resync_mb_y=mb_y;
2574                 }
2575             }
2576
2577             if(  (s->resync_mb_x   == s->mb_x)
2578                && s->resync_mb_y+1 == s->mb_y){
2579                 s->first_slice_line=0;
2580             }
2581
2582             s->mb_skipped=0;
2583             s->dquant=0; //only for QP_RD
2584
2585             update_mb_info(s, 0);
2586
2587             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
2588                 int next_block=0;
2589                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2590
2591                 copy_context_before_encode(&backup_s, s, -1);
2592                 backup_s.pb= s->pb;
2593                 best_s.data_partitioning= s->data_partitioning;
2594                 best_s.partitioned_frame= s->partitioned_frame;
2595                 if(s->data_partitioning){
2596                     backup_s.pb2= s->pb2;
2597                     backup_s.tex_pb= s->tex_pb;
2598                 }
2599
2600                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2601                     s->mv_dir = MV_DIR_FORWARD;
2602                     s->mv_type = MV_TYPE_16X16;
2603                     s->mb_intra= 0;
2604                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2605                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2606                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2607                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2608                 }
2609                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2610                     s->mv_dir = MV_DIR_FORWARD;
2611                     s->mv_type = MV_TYPE_FIELD;
2612                     s->mb_intra= 0;
2613                     for(i=0; i<2; i++){
2614                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2615                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2616                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2617                     }
2618                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2619                                  &dmin, &next_block, 0, 0);
2620                 }
2621                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2622                     s->mv_dir = MV_DIR_FORWARD;
2623                     s->mv_type = MV_TYPE_16X16;
2624                     s->mb_intra= 0;
2625                     s->mv[0][0][0] = 0;
2626                     s->mv[0][0][1] = 0;
2627                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2628                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2629                 }
2630                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2631                     s->mv_dir = MV_DIR_FORWARD;
2632                     s->mv_type = MV_TYPE_8X8;
2633                     s->mb_intra= 0;
2634                     for(i=0; i<4; i++){
2635                         s->mv[0][i][0] = s->current_picture.f.motion_val[0][s->block_index[i]][0];
2636                         s->mv[0][i][1] = s->current_picture.f.motion_val[0][s->block_index[i]][1];
2637                     }
2638                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2639                                  &dmin, &next_block, 0, 0);
2640                 }
2641                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2642                     s->mv_dir = MV_DIR_FORWARD;
2643                     s->mv_type = MV_TYPE_16X16;
2644                     s->mb_intra= 0;
2645                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2646                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2647                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2648                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2649                 }
2650                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2651                     s->mv_dir = MV_DIR_BACKWARD;
2652                     s->mv_type = MV_TYPE_16X16;
2653                     s->mb_intra= 0;
2654                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2655                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2656                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2657                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2658                 }
2659                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2660                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2661                     s->mv_type = MV_TYPE_16X16;
2662                     s->mb_intra= 0;
2663                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2664                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2665                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2666                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2667                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2668                                  &dmin, &next_block, 0, 0);
2669                 }
2670                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2671                     s->mv_dir = MV_DIR_FORWARD;
2672                     s->mv_type = MV_TYPE_FIELD;
2673                     s->mb_intra= 0;
2674                     for(i=0; i<2; i++){
2675                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2676                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2677                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2678                     }
2679                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2680                                  &dmin, &next_block, 0, 0);
2681                 }
2682                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2683                     s->mv_dir = MV_DIR_BACKWARD;
2684                     s->mv_type = MV_TYPE_FIELD;
2685                     s->mb_intra= 0;
2686                     for(i=0; i<2; i++){
2687                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2688                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2689                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2690                     }
2691                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2692                                  &dmin, &next_block, 0, 0);
2693                 }
2694                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2695                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2696                     s->mv_type = MV_TYPE_FIELD;
2697                     s->mb_intra= 0;
2698                     for(dir=0; dir<2; dir++){
2699                         for(i=0; i<2; i++){
2700                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2701                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2702                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2703                         }
2704                     }
2705                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2706                                  &dmin, &next_block, 0, 0);
2707                 }
2708                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2709                     s->mv_dir = 0;
2710                     s->mv_type = MV_TYPE_16X16;
2711                     s->mb_intra= 1;
2712                     s->mv[0][0][0] = 0;
2713                     s->mv[0][0][1] = 0;
2714                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2715                                  &dmin, &next_block, 0, 0);
2716                     if(s->h263_pred || s->h263_aic){
2717                         if(best_s.mb_intra)
2718                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2719                         else
2720                             ff_clean_intra_table_entries(s); //old mode?
2721                     }
2722                 }
2723
2724                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
2725                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2726                         const int last_qp= backup_s.qscale;
2727                         int qpi, qp, dc[6];
2728                         DCTELEM ac[6][16];
2729                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2730                         static const int dquant_tab[4]={-1,1,-2,2};
2731
2732                         assert(backup_s.dquant == 0);
2733
2734                         //FIXME intra
2735                         s->mv_dir= best_s.mv_dir;
2736                         s->mv_type = MV_TYPE_16X16;
2737                         s->mb_intra= best_s.mb_intra;
2738                         s->mv[0][0][0] = best_s.mv[0][0][0];
2739                         s->mv[0][0][1] = best_s.mv[0][0][1];
2740                         s->mv[1][0][0] = best_s.mv[1][0][0];
2741                         s->mv[1][0][1] = best_s.mv[1][0][1];
2742
2743                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
2744                         for(; qpi<4; qpi++){
2745                             int dquant= dquant_tab[qpi];
2746                             qp= last_qp + dquant;
2747                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
2748                                 continue;
2749                             backup_s.dquant= dquant;
2750                             if(s->mb_intra && s->dc_val[0]){
2751                                 for(i=0; i<6; i++){
2752                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
2753                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(DCTELEM)*16);
2754                                 }
2755                             }
2756
2757                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2758                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
2759                             if(best_s.qscale != qp){
2760                                 if(s->mb_intra && s->dc_val[0]){
2761                                     for(i=0; i<6; i++){
2762                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
2763                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(DCTELEM)*16);
2764                                     }
2765                                 }
2766                             }
2767                         }
2768                     }
2769                 }
2770                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
2771                     int mx= s->b_direct_mv_table[xy][0];
2772                     int my= s->b_direct_mv_table[xy][1];
2773
2774                     backup_s.dquant = 0;
2775                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2776                     s->mb_intra= 0;
2777                     ff_mpeg4_set_direct_mv(s, mx, my);
2778                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2779                                  &dmin, &next_block, mx, my);
2780                 }
2781                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
2782                     backup_s.dquant = 0;
2783                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
2784                     s->mb_intra= 0;
2785                     ff_mpeg4_set_direct_mv(s, 0, 0);
2786                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
2787                                  &dmin, &next_block, 0, 0);
2788                 }
2789                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
2790                     int coded=0;
2791                     for(i=0; i<6; i++)
2792                         coded |= s->block_last_index[i];
2793                     if(coded){
2794                         int mx,my;
2795                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
2796                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
2797                             mx=my=0; //FIXME find the one we actually used
2798                             ff_mpeg4_set_direct_mv(s, mx, my);
2799                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
2800                             mx= s->mv[1][0][0];
2801                             my= s->mv[1][0][1];
2802                         }else{
2803                             mx= s->mv[0][0][0];
2804                             my= s->mv[0][0][1];
2805                         }
2806
2807                         s->mv_dir= best_s.mv_dir;
2808                         s->mv_type = best_s.mv_type;
2809                         s->mb_intra= 0;
2810 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
2811                         s->mv[0][0][1] = best_s.mv[0][0][1];
2812                         s->mv[1][0][0] = best_s.mv[1][0][0];
2813                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
2814                         backup_s.dquant= 0;
2815                         s->skipdct=1;
2816                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
2817                                         &dmin, &next_block, mx, my);
2818                         s->skipdct=0;
2819                     }
2820                 }
2821
2822                 s->current_picture.f.qscale_table[xy] = best_s.qscale;
2823
2824                 copy_context_after_encode(s, &best_s, -1);
2825
2826                 pb_bits_count= put_bits_count(&s->pb);
2827                 flush_put_bits(&s->pb);
2828                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
2829                 s->pb= backup_s.pb;
2830
2831                 if(s->data_partitioning){
2832                     pb2_bits_count= put_bits_count(&s->pb2);
2833                     flush_put_bits(&s->pb2);
2834                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
2835                     s->pb2= backup_s.pb2;
2836
2837                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
2838                     flush_put_bits(&s->tex_pb);
2839                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
2840                     s->tex_pb= backup_s.tex_pb;
2841                 }
2842                 s->last_bits= put_bits_count(&s->pb);
2843
2844                 if (CONFIG_H263_ENCODER &&
2845                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2846                     ff_h263_update_motion_val(s);
2847
2848                 if(next_block==0){ //FIXME 16 vs linesize16
2849                     s->dsp.put_pixels_tab[0][0](s->dest[0], s->rd_scratchpad                     , s->linesize  ,16);
2850                     s->dsp.put_pixels_tab[1][0](s->dest[1], s->rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
2851                     s->dsp.put_pixels_tab[1][0](s->dest[2], s->rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
2852                 }
2853
2854                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
2855                     ff_MPV_decode_mb(s, s->block);
2856             } else {
2857                 int motion_x = 0, motion_y = 0;
2858                 s->mv_type=MV_TYPE_16X16;
2859                 // only one MB-Type possible
2860
2861                 switch(mb_type){
2862                 case CANDIDATE_MB_TYPE_INTRA:
2863                     s->mv_dir = 0;
2864                     s->mb_intra= 1;
2865                     motion_x= s->mv[0][0][0] = 0;
2866                     motion_y= s->mv[0][0][1] = 0;
2867                     break;
2868                 case CANDIDATE_MB_TYPE_INTER:
2869                     s->mv_dir = MV_DIR_FORWARD;
2870                     s->mb_intra= 0;
2871                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
2872                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
2873                     break;
2874                 case CANDIDATE_MB_TYPE_INTER_I:
2875                     s->mv_dir = MV_DIR_FORWARD;
2876                     s->mv_type = MV_TYPE_FIELD;
2877                     s->mb_intra= 0;
2878                     for(i=0; i<2; i++){
2879                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2880                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2881                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2882                     }
2883                     break;
2884                 case CANDIDATE_MB_TYPE_INTER4V:
2885                     s->mv_dir = MV_DIR_FORWARD;
2886                     s->mv_type = MV_TYPE_8X8;
2887                     s->mb_intra= 0;
2888                     for(i=0; i<4; i++){
2889                         s->mv[0][i][0] = s->current_picture.f.motion_val[0][s->block_index[i]][0];
2890                         s->mv[0][i][1] = s->current_picture.f.motion_val[0][s->block_index[i]][1];
2891                     }
2892                     break;
2893                 case CANDIDATE_MB_TYPE_DIRECT:
2894                     if (CONFIG_MPEG4_ENCODER) {
2895                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2896                         s->mb_intra= 0;
2897                         motion_x=s->b_direct_mv_table[xy][0];
2898                         motion_y=s->b_direct_mv_table[xy][1];
2899                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
2900                     }
2901                     break;
2902                 case CANDIDATE_MB_TYPE_DIRECT0:
2903                     if (CONFIG_MPEG4_ENCODER) {
2904                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
2905                         s->mb_intra= 0;
2906                         ff_mpeg4_set_direct_mv(s, 0, 0);
2907                     }
2908                     break;
2909                 case CANDIDATE_MB_TYPE_BIDIR:
2910                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2911                     s->mb_intra= 0;
2912                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2913                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2914                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2915                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2916                     break;
2917                 case CANDIDATE_MB_TYPE_BACKWARD:
2918                     s->mv_dir = MV_DIR_BACKWARD;
2919                     s->mb_intra= 0;
2920                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2921                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2922                     break;
2923                 case CANDIDATE_MB_TYPE_FORWARD:
2924                     s->mv_dir = MV_DIR_FORWARD;
2925                     s->mb_intra= 0;
2926                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2927                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2928 //                    printf(" %d %d ", motion_x, motion_y);
2929                     break;
2930                 case CANDIDATE_MB_TYPE_FORWARD_I:
2931                     s->mv_dir = MV_DIR_FORWARD;
2932                     s->mv_type = MV_TYPE_FIELD;
2933                     s->mb_intra= 0;
2934                     for(i=0; i<2; i++){
2935                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2936                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2937                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2938                     }
2939                     break;
2940                 case CANDIDATE_MB_TYPE_BACKWARD_I:
2941                     s->mv_dir = MV_DIR_BACKWARD;
2942                     s->mv_type = MV_TYPE_FIELD;
2943                     s->mb_intra= 0;
2944                     for(i=0; i<2; i++){
2945                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2946                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2947                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2948                     }
2949                     break;
2950                 case CANDIDATE_MB_TYPE_BIDIR_I:
2951                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2952                     s->mv_type = MV_TYPE_FIELD;
2953                     s->mb_intra= 0;
2954                     for(dir=0; dir<2; dir++){
2955                         for(i=0; i<2; i++){
2956                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2957                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2958                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2959                         }
2960                     }
2961                     break;
2962                 default:
2963                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
2964                 }
2965
2966                 encode_mb(s, motion_x, motion_y);
2967
2968                 // RAL: Update last macroblock type
2969                 s->last_mv_dir = s->mv_dir;
2970
2971                 if (CONFIG_H263_ENCODER &&
2972                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
2973                     ff_h263_update_motion_val(s);
2974
2975                 ff_MPV_decode_mb(s, s->block);
2976             }
2977
2978             /* clean the MV table in IPS frames for direct mode in B frames */
2979             if(s->mb_intra /* && I,P,S_TYPE */){
2980                 s->p_mv_table[xy][0]=0;
2981                 s->p_mv_table[xy][1]=0;
2982             }
2983
2984             if(s->flags&CODEC_FLAG_PSNR){
2985                 int w= 16;
2986                 int h= 16;
2987
2988                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2989                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2990
2991                 s->current_picture.f.error[0] += sse(
2992                     s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
2993                     s->dest[0], w, h, s->linesize);
2994                 s->current_picture.f.error[1] += sse(
2995                     s, s->new_picture.f.data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2996                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
2997                 s->current_picture.f.error[2] += sse(
2998                     s, s->new_picture.f.data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
2999                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3000             }
3001             if(s->loop_filter){
3002                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
3003                     ff_h263_loop_filter(s);
3004             }
3005 //printf("MB %d %d bits\n", s->mb_x+s->mb_y*s->mb_stride, put_bits_count(&s->pb));
3006         }
3007     }
3008
3009     //not beautiful here but we must write it before flushing so it has to be here
3010     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
3011         ff_msmpeg4_encode_ext_header(s);
3012
3013     write_slice_end(s);
3014
3015     /* Send the last GOB if RTP */
3016     if (s->avctx->rtp_callback) {
3017         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
3018         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
3019         /* Call the RTP callback to send the last GOB */
3020         emms_c();
3021         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
3022     }
3023
3024     return 0;
3025 }
3026
3027 #define MERGE(field) dst->field += src->field; src->field=0
3028 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
3029     MERGE(me.scene_change_score);
3030     MERGE(me.mc_mb_var_sum_temp);
3031     MERGE(me.mb_var_sum_temp);
3032 }
3033
3034 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
3035     int i;
3036
3037     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
3038     MERGE(dct_count[1]);
3039     MERGE(mv_bits);
3040     MERGE(i_tex_bits);
3041     MERGE(p_tex_bits);
3042     MERGE(i_count);
3043     MERGE(f_count);
3044     MERGE(b_count);
3045     MERGE(skip_count);
3046     MERGE(misc_bits);
3047     MERGE(error_count);
3048     MERGE(padding_bug_score);
3049     MERGE(current_picture.f.error[0]);
3050     MERGE(current_picture.f.error[1]);
3051     MERGE(current_picture.f.error[2]);
3052
3053     if(dst->avctx->noise_reduction){
3054         for(i=0; i<64; i++){
3055             MERGE(dct_error_sum[0][i]);
3056             MERGE(dct_error_sum[1][i]);
3057         }
3058     }
3059
3060     assert(put_bits_count(&src->pb) % 8 ==0);
3061     assert(put_bits_count(&dst->pb) % 8 ==0);
3062     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3063     flush_put_bits(&dst->pb);
3064 }
3065
3066 static int estimate_qp(MpegEncContext *s, int dry_run){
3067     if (s->next_lambda){
3068         s->current_picture_ptr->f.quality =
3069         s->current_picture.f.quality = s->next_lambda;
3070         if(!dry_run) s->next_lambda= 0;
3071     } else if (!s->fixed_qscale) {
3072         s->current_picture_ptr->f.quality =
3073         s->current_picture.f.quality = ff_rate_estimate_qscale(s, dry_run);
3074         if (s->current_picture.f.quality < 0)
3075             return -1;
3076     }
3077
3078     if(s->adaptive_quant){
3079         switch(s->codec_id){
3080         case AV_CODEC_ID_MPEG4:
3081             if (CONFIG_MPEG4_ENCODER)
3082                 ff_clean_mpeg4_qscales(s);
3083             break;
3084         case AV_CODEC_ID_H263:
3085         case AV_CODEC_ID_H263P:
3086         case AV_CODEC_ID_FLV1:
3087             if (CONFIG_H263_ENCODER)
3088                 ff_clean_h263_qscales(s);
3089             break;
3090         default:
3091             ff_init_qscale_tab(s);
3092         }
3093
3094         s->lambda= s->lambda_table[0];
3095         //FIXME broken
3096     }else
3097         s->lambda = s->current_picture.f.quality;
3098 //printf("%d %d\n", s->avctx->global_quality, s->current_picture.quality);
3099     update_qscale(s);
3100     return 0;
3101 }
3102
3103 /* must be called before writing the header */
3104 static void set_frame_distances(MpegEncContext * s){
3105     assert(s->current_picture_ptr->f.pts != AV_NOPTS_VALUE);
3106     s->time = s->current_picture_ptr->f.pts * s->avctx->time_base.num;
3107
3108     if(s->pict_type==AV_PICTURE_TYPE_B){
3109         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3110         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3111     }else{
3112         s->pp_time= s->time - s->last_non_b_time;
3113         s->last_non_b_time= s->time;
3114         assert(s->picture_number==0 || s->pp_time > 0);
3115     }
3116 }
3117
3118 static int encode_picture(MpegEncContext *s, int picture_number)
3119 {
3120     int i;
3121     int bits;
3122     int context_count = s->slice_context_count;
3123
3124     s->picture_number = picture_number;
3125
3126     /* Reset the average MB variance */
3127     s->me.mb_var_sum_temp    =
3128     s->me.mc_mb_var_sum_temp = 0;
3129
3130     /* we need to initialize some time vars before we can encode b-frames */
3131     // RAL: Condition added for MPEG1VIDEO
3132     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3133         set_frame_distances(s);
3134     if(CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4)
3135         ff_set_mpeg4_time(s);
3136
3137     s->me.scene_change_score=0;
3138
3139 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3140
3141     if(s->pict_type==AV_PICTURE_TYPE_I){
3142         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3143         else                        s->no_rounding=0;
3144     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3145         if(s->flipflop_rounding || s->codec_id == AV_CODEC_ID_H263P || s->codec_id == AV_CODEC_ID_MPEG4)
3146             s->no_rounding ^= 1;
3147     }
3148
3149     if(s->flags & CODEC_FLAG_PASS2){
3150         if (estimate_qp(s,1) < 0)
3151             return -1;
3152         ff_get_2pass_fcode(s);
3153     }else if(!(s->flags & CODEC_FLAG_QSCALE)){
3154         if(s->pict_type==AV_PICTURE_TYPE_B)
3155             s->lambda= s->last_lambda_for[s->pict_type];
3156         else
3157             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3158         update_qscale(s);
3159     }
3160
3161     s->mb_intra=0; //for the rate distortion & bit compare functions
3162     for(i=1; i<context_count; i++){
3163         ff_update_duplicate_context(s->thread_context[i], s);
3164     }
3165
3166     if(ff_init_me(s)<0)
3167         return -1;
3168
3169     /* Estimate motion for every MB */
3170     if(s->pict_type != AV_PICTURE_TYPE_I){
3171         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
3172         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
3173         if(s->pict_type != AV_PICTURE_TYPE_B && s->avctx->me_threshold==0){
3174             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
3175                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3176             }
3177         }
3178
3179         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3180     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3181         /* I-Frame */
3182         for(i=0; i<s->mb_stride*s->mb_height; i++)
3183             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3184
3185         if(!s->fixed_qscale){
3186             /* finding spatial complexity for I-frame rate control */
3187             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3188         }
3189     }
3190     for(i=1; i<context_count; i++){
3191         merge_context_after_me(s, s->thread_context[i]);
3192     }
3193     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3194     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3195     emms_c();
3196
3197     if(s->me.scene_change_score > s->avctx->scenechange_threshold && s->pict_type == AV_PICTURE_TYPE_P){
3198         s->pict_type= AV_PICTURE_TYPE_I;
3199         for(i=0; i<s->mb_stride*s->mb_height; i++)
3200             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3201 //printf("Scene change detected, encoding as I Frame %d %d\n", s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3202     }
3203
3204     if(!s->umvplus){
3205         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3206             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3207
3208             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3209                 int a,b;
3210                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3211                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3212                 s->f_code= FFMAX3(s->f_code, a, b);
3213             }
3214
3215             ff_fix_long_p_mvs(s);
3216             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3217             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3218                 int j;
3219                 for(i=0; i<2; i++){
3220                     for(j=0; j<2; j++)
3221                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3222                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3223                 }
3224             }
3225         }
3226
3227         if(s->pict_type==AV_PICTURE_TYPE_B){
3228             int a, b;
3229
3230             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3231             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3232             s->f_code = FFMAX(a, b);
3233
3234             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3235             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3236             s->b_code = FFMAX(a, b);
3237
3238             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3239             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3240             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3241             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3242             if(s->flags & CODEC_FLAG_INTERLACED_ME){
3243                 int dir, j;
3244                 for(dir=0; dir<2; dir++){
3245                     for(i=0; i<2; i++){
3246                         for(j=0; j<2; j++){
3247                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3248                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3249                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3250                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3251                         }
3252                     }
3253                 }
3254             }
3255         }
3256     }
3257
3258     if (estimate_qp(s, 0) < 0)
3259         return -1;
3260
3261     if(s->qscale < 3 && s->max_qcoeff<=128 && s->pict_type==AV_PICTURE_TYPE_I && !(s->flags & CODEC_FLAG_QSCALE))
3262         s->qscale= 3; //reduce clipping problems
3263
3264     if (s->out_format == FMT_MJPEG) {
3265         /* for mjpeg, we do include qscale in the matrix */
3266         for(i=1;i<64;i++){
3267             int j= s->dsp.idct_permutation[i];
3268
3269             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
3270         }
3271         s->y_dc_scale_table=
3272         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3273         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3274         ff_convert_matrix(&s->dsp, s->q_intra_matrix, s->q_intra_matrix16,
3275                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3276         s->qscale= 8;
3277     }
3278
3279     //FIXME var duplication
3280     s->current_picture_ptr->f.key_frame =
3281     s->current_picture.f.key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3282     s->current_picture_ptr->f.pict_type =
3283     s->current_picture.f.pict_type = s->pict_type;
3284
3285     if (s->current_picture.f.key_frame)
3286         s->picture_in_gop_number=0;
3287
3288     s->last_bits= put_bits_count(&s->pb);
3289     switch(s->out_format) {
3290     case FMT_MJPEG:
3291         if (CONFIG_MJPEG_ENCODER)
3292             ff_mjpeg_encode_picture_header(s);
3293         break;
3294     case FMT_H261:
3295         if (CONFIG_H261_ENCODER)
3296             ff_h261_encode_picture_header(s, picture_number);
3297         break;
3298     case FMT_H263:
3299         if (CONFIG_WMV2_ENCODER && s->codec_id == AV_CODEC_ID_WMV2)
3300             ff_wmv2_encode_picture_header(s, picture_number);
3301         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3302             ff_msmpeg4_encode_picture_header(s, picture_number);
3303         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
3304             ff_mpeg4_encode_picture_header(s, picture_number);
3305         else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10)
3306             ff_rv10_encode_picture_header(s, picture_number);
3307         else if (CONFIG_RV20_ENCODER && s->codec_id == AV_CODEC_ID_RV20)
3308             ff_rv20_encode_picture_header(s, picture_number);
3309         else if (CONFIG_FLV_ENCODER && s->codec_id == AV_CODEC_ID_FLV1)
3310             ff_flv_encode_picture_header(s, picture_number);
3311         else if (CONFIG_H263_ENCODER)
3312             ff_h263_encode_picture_header(s, picture_number);
3313         break;
3314     case FMT_MPEG1:
3315         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
3316             ff_mpeg1_encode_picture_header(s, picture_number);
3317         break;
3318     case FMT_H264:
3319         break;
3320     default:
3321         assert(0);
3322     }
3323     bits= put_bits_count(&s->pb);
3324     s->header_bits= bits - s->last_bits;
3325
3326     for(i=1; i<context_count; i++){
3327         update_duplicate_context_after_me(s->thread_context[i], s);
3328     }
3329     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3330     for(i=1; i<context_count; i++){
3331         merge_context_after_encode(s, s->thread_context[i]);
3332     }
3333     emms_c();
3334     return 0;
3335 }
3336
3337 static void denoise_dct_c(MpegEncContext *s, DCTELEM *block){
3338     const int intra= s->mb_intra;
3339     int i;
3340
3341     s->dct_count[intra]++;
3342
3343     for(i=0; i<64; i++){
3344         int level= block[i];
3345
3346         if(level){
3347             if(level>0){
3348                 s->dct_error_sum[intra][i] += level;
3349                 level -= s->dct_offset[intra][i];
3350                 if(level<0) level=0;
3351             }else{
3352                 s->dct_error_sum[intra][i] -= level;
3353                 level += s->dct_offset[intra][i];
3354                 if(level>0) level=0;
3355             }
3356             block[i]= level;
3357         }
3358     }
3359 }
3360
3361 static int dct_quantize_trellis_c(MpegEncContext *s,
3362                                   DCTELEM *block, int n,
3363                                   int qscale, int *overflow){
3364     const int *qmat;
3365     const uint8_t *scantable= s->intra_scantable.scantable;
3366     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3367     int max=0;
3368     unsigned int threshold1, threshold2;
3369     int bias=0;
3370     int run_tab[65];
3371     int level_tab[65];
3372     int score_tab[65];
3373     int survivor[65];
3374     int survivor_count;
3375     int last_run=0;
3376     int last_level=0;
3377     int last_score= 0;
3378     int last_i;
3379     int coeff[2][64];
3380     int coeff_count[64];
3381     int qmul, qadd, start_i, last_non_zero, i, dc;
3382     const int esc_length= s->ac_esc_length;
3383     uint8_t * length;
3384     uint8_t * last_length;
3385     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3386
3387     s->dsp.fdct (block);
3388
3389     if(s->dct_error_sum)
3390         s->denoise_dct(s, block);
3391     qmul= qscale*16;
3392     qadd= ((qscale-1)|1)*8;
3393
3394     if (s->mb_intra) {
3395         int q;
3396         if (!s->h263_aic) {
3397             if (n < 4)
3398                 q = s->y_dc_scale;
3399             else
3400                 q = s->c_dc_scale;
3401             q = q << 3;
3402         } else{
3403             /* For AIC we skip quant/dequant of INTRADC */
3404             q = 1 << 3;
3405             qadd=0;
3406         }
3407
3408         /* note: block[0] is assumed to be positive */
3409         block[0] = (block[0] + (q >> 1)) / q;
3410         start_i = 1;
3411         last_non_zero = 0;
3412         qmat = s->q_intra_matrix[qscale];
3413         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3414             bias= 1<<(QMAT_SHIFT-1);
3415         length     = s->intra_ac_vlc_length;
3416         last_length= s->intra_ac_vlc_last_length;
3417     } else {
3418         start_i = 0;
3419         last_non_zero = -1;
3420         qmat = s->q_inter_matrix[qscale];
3421         length     = s->inter_ac_vlc_length;
3422         last_length= s->inter_ac_vlc_last_length;
3423     }
3424     last_i= start_i;
3425
3426     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3427     threshold2= (threshold1<<1);
3428
3429     for(i=63; i>=start_i; i--) {
3430         const int j = scantable[i];
3431         int level = block[j] * qmat[j];
3432
3433         if(((unsigned)(level+threshold1))>threshold2){
3434             last_non_zero = i;
3435             break;
3436         }
3437     }
3438
3439     for(i=start_i; i<=last_non_zero; i++) {
3440         const int j = scantable[i];
3441         int level = block[j] * qmat[j];
3442
3443 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3444 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3445         if(((unsigned)(level+threshold1))>threshold2){
3446             if(level>0){
3447                 level= (bias + level)>>QMAT_SHIFT;
3448                 coeff[0][i]= level;
3449                 coeff[1][i]= level-1;
3450 //                coeff[2][k]= level-2;
3451             }else{
3452                 level= (bias - level)>>QMAT_SHIFT;
3453                 coeff[0][i]= -level;
3454                 coeff[1][i]= -level+1;
3455 //                coeff[2][k]= -level+2;
3456             }
3457             coeff_count[i]= FFMIN(level, 2);
3458             assert(coeff_count[i]);
3459             max |=level;
3460         }else{
3461             coeff[0][i]= (level>>31)|1;
3462             coeff_count[i]= 1;
3463         }
3464     }
3465
3466     *overflow= s->max_qcoeff < max; //overflow might have happened
3467
3468     if(last_non_zero < start_i){
3469         memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3470         return last_non_zero;
3471     }
3472
3473     score_tab[start_i]= 0;
3474     survivor[0]= start_i;
3475     survivor_count= 1;
3476
3477     for(i=start_i; i<=last_non_zero; i++){
3478         int level_index, j, zero_distortion;
3479         int dct_coeff= FFABS(block[ scantable[i] ]);
3480         int best_score=256*256*256*120;
3481
3482         if (s->dsp.fdct == ff_fdct_ifast)
3483             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3484         zero_distortion= dct_coeff*dct_coeff;
3485
3486         for(level_index=0; level_index < coeff_count[i]; level_index++){
3487             int distortion;
3488             int level= coeff[level_index][i];
3489             const int alevel= FFABS(level);
3490             int unquant_coeff;
3491
3492             assert(level);
3493
3494             if(s->out_format == FMT_H263){
3495                 unquant_coeff= alevel*qmul + qadd;
3496             }else{ //MPEG1
3497                 j= s->dsp.idct_permutation[ scantable[i] ]; //FIXME optimize
3498                 if(s->mb_intra){
3499                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3500                         unquant_coeff =   (unquant_coeff - 1) | 1;
3501                 }else{
3502                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3503                         unquant_coeff =   (unquant_coeff - 1) | 1;
3504                 }
3505                 unquant_coeff<<= 3;
3506             }
3507
3508             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3509             level+=64;
3510             if((level&(~127)) == 0){
3511                 for(j=survivor_count-1; j>=0; j--){
3512                     int run= i - survivor[j];
3513                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3514                     score += score_tab[i-run];
3515
3516                     if(score < best_score){
3517                         best_score= score;
3518                         run_tab[i+1]= run;
3519                         level_tab[i+1]= level-64;
3520                     }
3521                 }
3522
3523                 if(s->out_format == FMT_H263){
3524                     for(j=survivor_count-1; j>=0; j--){
3525                         int run= i - survivor[j];
3526                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3527                         score += score_tab[i-run];
3528                         if(score < last_score){
3529                             last_score= score;
3530                             last_run= run;
3531                             last_level= level-64;
3532                             last_i= i+1;
3533                         }
3534                     }
3535                 }
3536             }else{
3537                 distortion += esc_length*lambda;
3538                 for(j=survivor_count-1; j>=0; j--){
3539                     int run= i - survivor[j];
3540                     int score= distortion + score_tab[i-run];
3541
3542                     if(score < best_score){
3543                         best_score= score;
3544                         run_tab[i+1]= run;
3545                         level_tab[i+1]= level-64;
3546                     }
3547                 }
3548
3549                 if(s->out_format == FMT_H263){
3550                   for(j=survivor_count-1; j>=0; j--){
3551                         int run= i - survivor[j];
3552                         int score= distortion + score_tab[i-run];
3553                         if(score < last_score){
3554                             last_score= score;
3555                             last_run= run;
3556                             last_level= level-64;
3557                             last_i= i+1;
3558                         }
3559                     }
3560                 }
3561             }
3562         }
3563
3564         score_tab[i+1]= best_score;
3565
3566         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
3567         if(last_non_zero <= 27){
3568             for(; survivor_count; survivor_count--){
3569                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3570                     break;
3571             }
3572         }else{
3573             for(; survivor_count; survivor_count--){
3574                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3575                     break;
3576             }
3577         }
3578
3579         survivor[ survivor_count++ ]= i+1;
3580     }
3581
3582     if(s->out_format != FMT_H263){
3583         last_score= 256*256*256*120;
3584         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3585             int score= score_tab[i];
3586             if(i) score += lambda*2; //FIXME exacter?
3587
3588             if(score < last_score){
3589                 last_score= score;
3590                 last_i= i;
3591                 last_level= level_tab[i];
3592                 last_run= run_tab[i];
3593             }
3594         }
3595     }
3596
3597     s->coded_score[n] = last_score;
3598
3599     dc= FFABS(block[0]);
3600     last_non_zero= last_i - 1;
3601     memset(block + start_i, 0, (64-start_i)*sizeof(DCTELEM));
3602
3603     if(last_non_zero < start_i)
3604         return last_non_zero;
3605
3606     if(last_non_zero == 0 && start_i == 0){
3607         int best_level= 0;
3608         int best_score= dc * dc;
3609
3610         for(i=0; i<coeff_count[0]; i++){
3611             int level= coeff[i][0];
3612             int alevel= FFABS(level);
3613             int unquant_coeff, score, distortion;
3614
3615             if(s->out_format == FMT_H263){
3616                     unquant_coeff= (alevel*qmul + qadd)>>3;
3617             }else{ //MPEG1
3618                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3619                     unquant_coeff =   (unquant_coeff - 1) | 1;
3620             }
3621             unquant_coeff = (unquant_coeff + 4) >> 3;
3622             unquant_coeff<<= 3 + 3;
3623
3624             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3625             level+=64;
3626             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3627             else                    score= distortion + esc_length*lambda;
3628
3629             if(score < best_score){
3630                 best_score= score;
3631                 best_level= level - 64;
3632             }
3633         }
3634         block[0]= best_level;
3635         s->coded_score[n] = best_score - dc*dc;
3636         if(best_level == 0) return -1;
3637         else                return last_non_zero;
3638     }
3639
3640     i= last_i;
3641     assert(last_level);
3642
3643     block[ perm_scantable[last_non_zero] ]= last_level;
3644     i -= last_run + 1;
3645
3646     for(; i>start_i; i -= run_tab[i] + 1){
3647         block[ perm_scantable[i-1] ]= level_tab[i];
3648     }
3649
3650     return last_non_zero;
3651 }
3652
3653 //#define REFINE_STATS 1
3654 static int16_t basis[64][64];
3655
3656 static void build_basis(uint8_t *perm){
3657     int i, j, x, y;
3658     emms_c();
3659     for(i=0; i<8; i++){
3660         for(j=0; j<8; j++){
3661             for(y=0; y<8; y++){
3662                 for(x=0; x<8; x++){
3663                     double s= 0.25*(1<<BASIS_SHIFT);
3664                     int index= 8*i + j;
3665                     int perm_index= perm[index];
3666                     if(i==0) s*= sqrt(0.5);
3667                     if(j==0) s*= sqrt(0.5);
3668                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3669                 }
3670             }
3671         }
3672     }
3673 }
3674
3675 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3676                         DCTELEM *block, int16_t *weight, DCTELEM *orig,
3677                         int n, int qscale){
3678     int16_t rem[64];
3679     LOCAL_ALIGNED_16(DCTELEM, d1, [64]);
3680     const uint8_t *scantable= s->intra_scantable.scantable;
3681     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3682 //    unsigned int threshold1, threshold2;
3683 //    int bias=0;
3684     int run_tab[65];
3685     int prev_run=0;
3686     int prev_level=0;
3687     int qmul, qadd, start_i, last_non_zero, i, dc;
3688     uint8_t * length;
3689     uint8_t * last_length;
3690     int lambda;
3691     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3692 #ifdef REFINE_STATS
3693 static int count=0;
3694 static int after_last=0;
3695 static int to_zero=0;
3696 static int from_zero=0;
3697 static int raise=0;
3698 static int lower=0;
3699 static int messed_sign=0;
3700 #endif
3701
3702     if(basis[0][0] == 0)
3703         build_basis(s->dsp.idct_permutation);
3704
3705     qmul= qscale*2;
3706     qadd= (qscale-1)|1;
3707     if (s->mb_intra) {
3708         if (!s->h263_aic) {
3709             if (n < 4)
3710                 q = s->y_dc_scale;
3711             else
3712                 q = s->c_dc_scale;
3713         } else{
3714             /* For AIC we skip quant/dequant of INTRADC */
3715             q = 1;
3716             qadd=0;
3717         }
3718         q <<= RECON_SHIFT-3;
3719         /* note: block[0] is assumed to be positive */
3720         dc= block[0]*q;
3721 //        block[0] = (block[0] + (q >> 1)) / q;
3722         start_i = 1;
3723 //        if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3724 //            bias= 1<<(QMAT_SHIFT-1);
3725         length     = s->intra_ac_vlc_length;
3726         last_length= s->intra_ac_vlc_last_length;
3727     } else {
3728         dc= 0;
3729         start_i = 0;
3730         length     = s->inter_ac_vlc_length;
3731         last_length= s->inter_ac_vlc_last_length;
3732     }
3733     last_non_zero = s->block_last_index[n];
3734
3735 #ifdef REFINE_STATS
3736 {START_TIMER
3737 #endif
3738     dc += (1<<(RECON_SHIFT-1));
3739     for(i=0; i<64; i++){
3740         rem[i]= dc - (orig[i]<<RECON_SHIFT); //FIXME  use orig dirrectly instead of copying to rem[]
3741     }
3742 #ifdef REFINE_STATS
3743 STOP_TIMER("memset rem[]")}
3744 #endif
3745     sum=0;
3746     for(i=0; i<64; i++){
3747         int one= 36;
3748         int qns=4;
3749         int w;
3750
3751         w= FFABS(weight[i]) + qns*one;
3752         w= 15 + (48*qns*one + w/2)/w; // 16 .. 63
3753
3754         weight[i] = w;
3755 //        w=weight[i] = (63*qns + (w/2)) / w;
3756
3757         assert(w>0);
3758         assert(w<(1<<6));
3759         sum += w*w;
3760     }
3761     lambda= sum*(uint64_t)s->lambda2 >> (FF_LAMBDA_SHIFT - 6 + 6 + 6 + 6);
3762 #ifdef REFINE_STATS
3763 {START_TIMER
3764 #endif
3765     run=0;
3766     rle_index=0;
3767     for(i=start_i; i<=last_non_zero; i++){
3768         int j= perm_scantable[i];
3769         const int level= block[j];
3770         int coeff;
3771
3772         if(level){
3773             if(level<0) coeff= qmul*level - qadd;
3774             else        coeff= qmul*level + qadd;
3775             run_tab[rle_index++]=run;
3776             run=0;
3777
3778             s->dsp.add_8x8basis(rem, basis[j], coeff);
3779         }else{
3780             run++;
3781         }
3782     }
3783 #ifdef REFINE_STATS
3784 if(last_non_zero>0){
3785 STOP_TIMER("init rem[]")
3786 }
3787 }
3788
3789 {START_TIMER
3790 #endif
3791     for(;;){
3792         int best_score=s->dsp.try_8x8basis(rem, weight, basis[0], 0);
3793         int best_coeff=0;
3794         int best_change=0;
3795         int run2, best_unquant_change=0, analyze_gradient;
3796 #ifdef REFINE_STATS
3797 {START_TIMER
3798 #endif
3799         analyze_gradient = last_non_zero > 2 || s->quantizer_noise_shaping >= 3;
3800
3801         if(analyze_gradient){
3802 #ifdef REFINE_STATS
3803 {START_TIMER
3804 #endif
3805             for(i=0; i<64; i++){
3806                 int w= weight[i];
3807
3808                 d1[i] = (rem[i]*w*w + (1<<(RECON_SHIFT+12-1)))>>(RECON_SHIFT+12);
3809             }
3810 #ifdef REFINE_STATS
3811 STOP_TIMER("rem*w*w")}
3812 {START_TIMER
3813 #endif
3814             s->dsp.fdct(d1);
3815 #ifdef REFINE_STATS
3816 STOP_TIMER("dct")}
3817 #endif
3818         }
3819
3820         if(start_i){
3821             const int level= block[0];
3822             int change, old_coeff;
3823
3824             assert(s->mb_intra);
3825
3826             old_coeff= q*level;
3827
3828             for(change=-1; change<=1; change+=2){
3829                 int new_level= level + change;
3830                 int score, new_coeff;
3831
3832                 new_coeff= q*new_level;
3833                 if(new_coeff >= 2048 || new_coeff < 0)
3834                     continue;
3835
3836                 score= s->dsp.try_8x8basis(rem, weight, basis[0], new_coeff - old_coeff);
3837                 if(score<best_score){
3838                     best_score= score;
3839                     best_coeff= 0;
3840                     best_change= change;
3841                     best_unquant_change= new_coeff - old_coeff;
3842                 }
3843             }
3844         }
3845
3846         run=0;
3847         rle_index=0;
3848         run2= run_tab[rle_index++];
3849         prev_level=0;
3850         prev_run=0;
3851
3852         for(i=start_i; i<64; i++){
3853             int j= perm_scantable[i];
3854             const int level= block[j];
3855             int change, old_coeff;
3856
3857             if(s->quantizer_noise_shaping < 3 && i > last_non_zero + 1)
3858                 break;
3859
3860             if(level){
3861                 if(level<0) old_coeff= qmul*level - qadd;
3862                 else        old_coeff= qmul*level + qadd;
3863                 run2= run_tab[rle_index++]; //FIXME ! maybe after last
3864             }else{
3865                 old_coeff=0;
3866                 run2--;
3867                 assert(run2>=0 || i >= last_non_zero );
3868             }
3869
3870             for(change=-1; change<=1; change+=2){
3871                 int new_level= level + change;
3872                 int score, new_coeff, unquant_change;
3873
3874                 score=0;
3875                 if(s->quantizer_noise_shaping < 2 && FFABS(new_level) > FFABS(level))
3876                    continue;
3877
3878                 if(new_level){
3879                     if(new_level<0) new_coeff= qmul*new_level - qadd;