Add partially accelerated mc_luma
[x262.git] / encoder / macroblock.c
1 /*****************************************************************************
2  * macroblock.c: macroblock encoding
3  *****************************************************************************
4  * Copyright (C) 2003-2014 x264 project
5  *
6  * Authors: Laurent Aimar <fenrir@via.ecp.fr>
7  *          Loren Merritt <lorenm@u.washington.edu>
8  *          Jason Garrett-Glaser <darkshikari@gmail.com>
9  *          Henrik Gramner <henrik@gramner.com>
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2 of the License, or
14  * (at your option) any later version.
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19  * GNU General Public License for more details.
20  *
21  * You should have received a copy of the GNU General Public License
22  * along with this program; if not, write to the Free Software
23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
24  *
25  * This program is also available under a commercial proprietary license.
26  * For more information, contact us at licensing@x264.com.
27  *****************************************************************************/
28
29 #include "common/common.h"
30 #include "macroblock.h"
31
32 /* These chroma DC functions don't have assembly versions and are only used here. */
33
34 #define ZIG(i,y,x) level[i] = dct[x*2+y];
35 static inline void zigzag_scan_2x2_dc( dctcoef level[4], dctcoef dct[4] )
36 {
37     ZIG(0,0,0)
38     ZIG(1,0,1)
39     ZIG(2,1,0)
40     ZIG(3,1,1)
41 }
42 #undef ZIG
43
44 static inline void zigzag_scan_2x4_dc( dctcoef level[8], dctcoef dct[8] )
45 {
46     level[0] = dct[0];
47     level[1] = dct[2];
48     level[2] = dct[1];
49     level[3] = dct[4];
50     level[4] = dct[6];
51     level[5] = dct[3];
52     level[6] = dct[5];
53     level[7] = dct[7];
54 }
55
56 #define IDCT_DEQUANT_2X2_START \
57     int d0 = dct[0] + dct[1]; \
58     int d1 = dct[2] + dct[3]; \
59     int d2 = dct[0] - dct[1]; \
60     int d3 = dct[2] - dct[3]; \
61     int dmf = dequant_mf[i_qp%6][0] << i_qp/6;
62
63 static inline void idct_dequant_2x2_dc( dctcoef dct[4], dctcoef dct4x4[4][16], int dequant_mf[6][16], int i_qp )
64 {
65     IDCT_DEQUANT_2X2_START
66     dct4x4[0][0] = (d0 + d1) * dmf >> 5;
67     dct4x4[1][0] = (d0 - d1) * dmf >> 5;
68     dct4x4[2][0] = (d2 + d3) * dmf >> 5;
69     dct4x4[3][0] = (d2 - d3) * dmf >> 5;
70 }
71
72 static inline void idct_dequant_2x2_dconly( dctcoef dct[4], int dequant_mf[6][16], int i_qp )
73 {
74     IDCT_DEQUANT_2X2_START
75     dct[0] = (d0 + d1) * dmf >> 5;
76     dct[1] = (d0 - d1) * dmf >> 5;
77     dct[2] = (d2 + d3) * dmf >> 5;
78     dct[3] = (d2 - d3) * dmf >> 5;
79 }
80 #undef IDCT_2X2_DEQUANT_START
81
82 static inline void dct2x2dc( dctcoef d[4], dctcoef dct4x4[4][16] )
83 {
84     int d0 = dct4x4[0][0] + dct4x4[1][0];
85     int d1 = dct4x4[2][0] + dct4x4[3][0];
86     int d2 = dct4x4[0][0] - dct4x4[1][0];
87     int d3 = dct4x4[2][0] - dct4x4[3][0];
88     d[0] = d0 + d1;
89     d[2] = d2 + d3;
90     d[1] = d0 - d1;
91     d[3] = d2 - d3;
92     dct4x4[0][0] = 0;
93     dct4x4[1][0] = 0;
94     dct4x4[2][0] = 0;
95     dct4x4[3][0] = 0;
96 }
97
98 static ALWAYS_INLINE int array_non_zero( dctcoef *v, int i_count )
99 {
100     if( WORD_SIZE == 8 )
101     {
102         for( int i = 0; i < i_count; i += 8/sizeof(dctcoef) )
103             if( M64( &v[i] ) )
104                 return 1;
105     }
106     else
107     {
108         for( int i = 0; i < i_count; i += 4/sizeof(dctcoef) )
109             if( M32( &v[i] ) )
110                 return 1;
111     }
112     return 0;
113 }
114
115 /* All encoding functions must output the correct CBP and NNZ values.
116  * The entropy coding functions will check CBP first, then NNZ, before
117  * actually reading the DCT coefficients.  NNZ still must be correct even
118  * if CBP is zero because of the use of NNZ values for context selection.
119  * "NNZ" need only be 0 or 1 rather than the exact coefficient count because
120  * that is only needed in CAVLC, and will be calculated by CAVLC's residual
121  * coding and stored as necessary. */
122
123 /* This means that decimation can be done merely by adjusting the CBP and NNZ
124  * rather than memsetting the coefficients. */
125
126 static void x264_mb_encode_i16x16( x264_t *h, int p, int i_qp )
127 {
128     pixel *p_src = h->mb.pic.p_fenc[p];
129     pixel *p_dst = h->mb.pic.p_fdec[p];
130
131     ALIGNED_ARRAY_N( dctcoef, dct4x4,[16],[16] );
132     ALIGNED_ARRAY_N( dctcoef, dct_dc4x4,[16] );
133
134     int nz, block_cbp = 0;
135     int decimate_score = h->mb.b_dct_decimate ? 0 : 9;
136     int i_quant_cat = p ? CQM_4IC : CQM_4IY;
137     int i_mode = h->mb.i_intra16x16_pred_mode;
138
139     if( h->mb.b_lossless )
140         x264_predict_lossless_16x16( h, p, i_mode );
141     else
142         h->predict_16x16[i_mode]( h->mb.pic.p_fdec[p] );
143
144     if( h->mb.b_lossless )
145     {
146         for( int i = 0; i < 16; i++ )
147         {
148             int oe = block_idx_xy_fenc[i];
149             int od = block_idx_xy_fdec[i];
150             nz = h->zigzagf.sub_4x4ac( h->dct.luma4x4[16*p+i], p_src+oe, p_dst+od, &dct_dc4x4[block_idx_yx_1d[i]] );
151             h->mb.cache.non_zero_count[x264_scan8[16*p+i]] = nz;
152             block_cbp |= nz;
153         }
154         h->mb.i_cbp_luma |= block_cbp * 0xf;
155         h->mb.cache.non_zero_count[x264_scan8[LUMA_DC+p]] = array_non_zero( dct_dc4x4, 16 );
156         h->zigzagf.scan_4x4( h->dct.luma16x16_dc[p], dct_dc4x4 );
157         return;
158     }
159
160     CLEAR_16x16_NNZ( p );
161
162     h->dctf.sub16x16_dct( dct4x4, p_src, p_dst );
163
164     if( h->mb.b_noise_reduction )
165         for( int idx = 0; idx < 16; idx++ )
166             h->quantf.denoise_dct( dct4x4[idx], h->nr_residual_sum[0], h->nr_offset[0], 16 );
167
168     for( int idx = 0; idx < 16; idx++ )
169     {
170         dct_dc4x4[block_idx_xy_1d[idx]] = dct4x4[idx][0];
171         dct4x4[idx][0] = 0;
172     }
173
174     if( h->mb.b_trellis )
175     {
176         for( int idx = 0; idx < 16; idx++ )
177             if( x264_quant_4x4_trellis( h, dct4x4[idx], i_quant_cat, i_qp, ctx_cat_plane[DCT_LUMA_AC][p], 1, !!p, idx ) )
178             {
179                 block_cbp = 0xf;
180                 h->zigzagf.scan_4x4( h->dct.luma4x4[16*p+idx], dct4x4[idx] );
181                 h->quantf.dequant_4x4( dct4x4[idx], h->dequant4_mf[i_quant_cat], i_qp );
182                 if( decimate_score < 6 ) decimate_score += h->quantf.decimate_score15( h->dct.luma4x4[16*p+idx] );
183                 h->mb.cache.non_zero_count[x264_scan8[16*p+idx]] = 1;
184             }
185     }
186     else
187     {
188         for( int i8x8 = 0; i8x8 < 4; i8x8++ )
189         {
190             nz = h->quantf.quant_4x4x4( &dct4x4[i8x8*4], h->quant4_mf[i_quant_cat][i_qp], h->quant4_bias[i_quant_cat][i_qp] );
191             if( nz )
192             {
193                 block_cbp = 0xf;
194                 FOREACH_BIT( idx, i8x8*4, nz )
195                 {
196                     h->zigzagf.scan_4x4( h->dct.luma4x4[16*p+idx], dct4x4[idx] );
197                     h->quantf.dequant_4x4( dct4x4[idx], h->dequant4_mf[i_quant_cat], i_qp );
198                     if( decimate_score < 6 ) decimate_score += h->quantf.decimate_score15( h->dct.luma4x4[16*p+idx] );
199                     h->mb.cache.non_zero_count[x264_scan8[16*p+idx]] = 1;
200                 }
201             }
202         }
203     }
204
205     /* Writing the 16 CBFs in an i16x16 block is quite costly, so decimation can save many bits. */
206     /* More useful with CAVLC, but still useful with CABAC. */
207     if( decimate_score < 6 )
208     {
209         CLEAR_16x16_NNZ( p );
210         block_cbp = 0;
211     }
212     else
213         h->mb.i_cbp_luma |= block_cbp;
214
215     h->dctf.dct4x4dc( dct_dc4x4 );
216     if( h->mb.b_trellis )
217         nz = x264_quant_luma_dc_trellis( h, dct_dc4x4, i_quant_cat, i_qp, ctx_cat_plane[DCT_LUMA_DC][p], 1, LUMA_DC+p );
218     else
219         nz = h->quantf.quant_4x4_dc( dct_dc4x4, h->quant4_mf[i_quant_cat][i_qp][0]>>1, h->quant4_bias[i_quant_cat][i_qp][0]<<1 );
220
221     h->mb.cache.non_zero_count[x264_scan8[LUMA_DC+p]] = nz;
222     if( nz )
223     {
224         h->zigzagf.scan_4x4( h->dct.luma16x16_dc[p], dct_dc4x4 );
225
226         /* output samples to fdec */
227         h->dctf.idct4x4dc( dct_dc4x4 );
228         h->quantf.dequant_4x4_dc( dct_dc4x4, h->dequant4_mf[i_quant_cat], i_qp );  /* XXX not inversed */
229         if( block_cbp )
230             for( int i = 0; i < 16; i++ )
231                 dct4x4[i][0] = dct_dc4x4[block_idx_xy_1d[i]];
232     }
233
234     /* put pixels to fdec */
235     if( block_cbp )
236         h->dctf.add16x16_idct( p_dst, dct4x4 );
237     else if( nz )
238         h->dctf.add16x16_idct_dc( p_dst, dct_dc4x4 );
239
240 }
241
242 static void x264_mb_encode_intra_block_mpeg2( x264_t *h, int idx, int i_qp )
243 {
244     pixel *p_src;
245     pixel *p_dst;
246     ALIGNED_ARRAY_16( dctcoef, dct8x8,[64] );
247     int nz, cur_dc_predictor, dc_diff, size;
248     int chroma422 = ( CHROMA_FORMAT == CHROMA_422 && idx > 3 ) ? 2 : 0;
249     int chroma = idx > 3 ? 1 : 0;
250
251     // TODO decimation
252
253     int x = idx&1;
254     if( idx < 4 )
255     {
256         int y = idx>>1;
257         p_src = &h->mb.pic.p_fenc[0][8*x + 8*y*FENC_STRIDE];
258         p_dst = &h->mb.pic.p_fdec[0][8*x + 8*y*FDEC_STRIDE];
259         cur_dc_predictor = idx == 0 ? h->mb.i_intra_dc_predictor[3] : h->mb.i_intra_dc_predictor[idx-1];
260     }
261     else if( idx < 6)
262     {
263         p_src = h->mb.pic.p_fenc[1+x];
264         p_dst = h->mb.pic.p_fdec[1+x];
265         cur_dc_predictor = h->mb.i_intra_dc_predictor[4+x+chroma422];
266     }
267     else // CHROMA_422
268     {
269         p_src = &h->mb.pic.p_fenc[1+x][8*FENC_STRIDE];
270         p_dst = &h->mb.pic.p_fdec[1+x][8*FDEC_STRIDE];
271         cur_dc_predictor = h->mb.i_intra_dc_predictor[4+x];
272     }
273
274     h->dctf.sub8x8_dct8( dct8x8, p_src, p_dst );
275
276     h->nr_count[chroma] += h->mb.b_noise_reduction;
277     if( h->mb.b_noise_reduction)
278         h->quantf.denoise_dct( dct8x8, h->nr_residual_sum[chroma], h->nr_offset[chroma], 64 );
279
280     nz = h->quantf.quant_8x8( dct8x8, h->quant8_mf[CQM_8IY+chroma422][i_qp],
281                               h->quant8_bias[CQM_8IY+chroma422][i_qp] );
282
283     // DC prediction
284     dc_diff = dct8x8[0] - cur_dc_predictor;
285     if( dc_diff < 0 )
286     {
287         size = LOG2_16( -2*dc_diff );
288         dc_diff--;
289     }
290     else
291         size = LOG2_16( 2*dc_diff );
292     h->mb.i_dct_dc_size[idx] = size;
293     h->mb.i_dct_dc_diff[idx] = dc_diff & ((1<<size)-1);
294     h->mb.i_intra_dc_predictor[idx] = dct8x8[0];
295
296     if( nz )
297     {
298         if( idx < 4 )
299             h->mb.i_cbp_luma |= 1<<(3-idx);
300         else if( idx < 6 )
301             h->mb.i_cbp_chroma |= 1<<(5-idx);
302         else
303             h->mb.i_cbp_chroma422 |= 1<<(7-idx);
304         h->zigzagf.scan_8x8( h->dct.mpeg2_8x8[idx], dct8x8 );
305         h->quantf.dequant_mpeg2_intra( dct8x8, h->dequant8_mf[CQM_8IY+chroma422][i_qp],
306                                        h->param.i_intra_dc_precision );
307         h->dctf.add8x8_idct8( p_dst, dct8x8 );
308     }
309 }
310
311 static void x264_mb_encode_inter_block_mpeg2( x264_t *h, int idx, int i_qp )
312 {
313     pixel *p_src;
314     pixel *p_dst;
315     ALIGNED_ARRAY_16( dctcoef, dct8x8,[64] );
316     int chroma422 = ( CHROMA_FORMAT == CHROMA_422 && idx > 3 ) ? 2 : 0;
317     int chroma = idx > 3 ? 1 : 0;
318
319     int x = idx&1;
320     if( idx < 4 )
321     {
322         int y = idx>>1;
323         p_src = &h->mb.pic.p_fenc[0][8*x + 8*y*FENC_STRIDE];
324         p_dst = &h->mb.pic.p_fdec[0][8*x + 8*y*FDEC_STRIDE];
325     }
326     else if( idx < 6 )
327     {
328         p_src = h->mb.pic.p_fenc[1+x];
329         p_dst = h->mb.pic.p_fdec[1+x];
330     }
331     else // CHROMA_422
332     {
333         p_src = &h->mb.pic.p_fenc[1+x][8*FENC_STRIDE];
334         p_dst = &h->mb.pic.p_fdec[1+x][8*FDEC_STRIDE];
335     }
336     h->dctf.sub8x8_dct8( dct8x8, p_src, p_dst );
337
338     h->nr_count[chroma] += h->mb.b_noise_reduction;
339     if( h->mb.b_noise_reduction)
340         h->quantf.denoise_dct( dct8x8, h->nr_residual_sum[chroma], h->nr_offset[chroma], 64 );
341
342     int nz = h->quantf.quant_8x8( dct8x8, h->quant8_mf[CQM_8PY+chroma422][i_qp],
343                                           h->quant8_bias[CQM_8PY+chroma422][i_qp] );
344     if( nz )
345     {
346         if( idx < 4 )
347             h->mb.i_cbp_luma |= 1<<(3-idx);
348         else if( idx < 6 )
349             h->mb.i_cbp_chroma |= 1<<(5-idx);
350         else
351             h->mb.i_cbp_chroma422 |= 1<<(7-idx);
352         h->zigzagf.scan_8x8( h->dct.mpeg2_8x8[idx], dct8x8 );
353         h->quantf.dequant_mpeg2_inter( dct8x8, h->dequant8_mf[CQM_8PY+chroma422][i_qp] );
354         h->dctf.add8x8_idct8( p_dst, dct8x8 );
355     }
356 }
357
358 /* Round down coefficients losslessly in DC-only chroma blocks.
359  * Unlike luma blocks, this can't be done with a lookup table or
360  * other shortcut technique because of the interdependencies
361  * between the coefficients due to the chroma DC transform. */
362 static ALWAYS_INLINE int x264_mb_optimize_chroma_dc( x264_t *h, dctcoef *dct_dc, int dequant_mf[6][16], int i_qp, int chroma422 )
363 {
364     int dmf = dequant_mf[i_qp%6][0] << i_qp/6;
365
366     /* If the QP is too high, there's no benefit to rounding optimization. */
367     if( dmf > 32*64 )
368         return 1;
369
370     if( chroma422 )
371         return h->quantf.optimize_chroma_2x4_dc( dct_dc, dmf );
372     else
373         return h->quantf.optimize_chroma_2x2_dc( dct_dc, dmf );
374 }
375
376 static ALWAYS_INLINE void x264_mb_encode_chroma_internal( x264_t *h, int b_inter, int i_qp, int chroma422 )
377 {
378     int nz, nz_dc;
379     int b_decimate = b_inter && h->mb.b_dct_decimate;
380     int (*dequant_mf)[16] = h->dequant4_mf[CQM_4IC + b_inter];
381     ALIGNED_ARRAY_16( dctcoef, dct_dc,[8] );
382     h->mb.i_cbp_chroma = 0;
383     h->nr_count[2] += h->mb.b_noise_reduction * 4;
384
385     M16( &h->mb.cache.non_zero_count[x264_scan8[16]] ) = 0;
386     M16( &h->mb.cache.non_zero_count[x264_scan8[18]] ) = 0;
387     M16( &h->mb.cache.non_zero_count[x264_scan8[32]] ) = 0;
388     M16( &h->mb.cache.non_zero_count[x264_scan8[34]] ) = 0;
389     if( chroma422 )
390     {
391         M16( &h->mb.cache.non_zero_count[x264_scan8[24]] ) = 0;
392         M16( &h->mb.cache.non_zero_count[x264_scan8[26]] ) = 0;
393         M16( &h->mb.cache.non_zero_count[x264_scan8[40]] ) = 0;
394         M16( &h->mb.cache.non_zero_count[x264_scan8[42]] ) = 0;
395     }
396
397     /* Early termination: check variance of chroma residual before encoding.
398      * Don't bother trying early termination at low QPs.
399      * Values are experimentally derived. */
400     if( b_decimate && i_qp >= (h->mb.b_trellis ? 12 : 18) && !h->mb.b_noise_reduction )
401     {
402         int thresh = chroma422 ? (x264_lambda2_tab[i_qp] + 16) >> 5 : (x264_lambda2_tab[i_qp] + 32) >> 6;
403         int ssd[2];
404         int chromapix = chroma422 ? PIXEL_8x16 : PIXEL_8x8;
405
406         int score  = h->pixf.var2[chromapix]( h->mb.pic.p_fenc[1], FENC_STRIDE, h->mb.pic.p_fdec[1], FDEC_STRIDE, &ssd[0] );
407         if( score < thresh*4 )
408             score += h->pixf.var2[chromapix]( h->mb.pic.p_fenc[2], FENC_STRIDE, h->mb.pic.p_fdec[2], FDEC_STRIDE, &ssd[1] );
409         if( score < thresh*4 )
410         {
411             h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+0]] = 0;
412             h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+1]] = 0;
413
414             for( int ch = 0; ch < 2; ch++ )
415             {
416                 if( ssd[ch] > thresh )
417                 {
418                     pixel *p_src = h->mb.pic.p_fenc[1+ch];
419                     pixel *p_dst = h->mb.pic.p_fdec[1+ch];
420
421                     if( chroma422 )
422                         /* Cannot be replaced by two calls to sub8x8_dct_dc since the hadamard transform is different */
423                         h->dctf.sub8x16_dct_dc( dct_dc, p_src, p_dst );
424                     else
425                         h->dctf.sub8x8_dct_dc( dct_dc, p_src, p_dst );
426
427                     if( h->mb.b_trellis )
428                         nz_dc = x264_quant_chroma_dc_trellis( h, dct_dc, i_qp+3*chroma422, !b_inter, CHROMA_DC+ch );
429                     else
430                     {
431                         nz_dc = 0;
432                         for( int i = 0; i <= chroma422; i++ )
433                             nz_dc |= h->quantf.quant_2x2_dc( &dct_dc[4*i], h->quant4_mf[CQM_4IC+b_inter][i_qp+3*chroma422][0] >> 1,
434                                                              h->quant4_bias[CQM_4IC+b_inter][i_qp+3*chroma422][0] << 1 );
435                     }
436
437                     if( nz_dc )
438                     {
439                         if( !x264_mb_optimize_chroma_dc( h, dct_dc, dequant_mf, i_qp+3*chroma422, chroma422 ) )
440                             continue;
441                         h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+ch]] = 1;
442                         if( chroma422 )
443                         {
444                             zigzag_scan_2x4_dc( h->dct.chroma_dc[ch], dct_dc );
445                             h->quantf.idct_dequant_2x4_dconly( dct_dc, dequant_mf, i_qp+3 );
446                         }
447                         else
448                         {
449                             zigzag_scan_2x2_dc( h->dct.chroma_dc[ch], dct_dc );
450                             idct_dequant_2x2_dconly( dct_dc, dequant_mf, i_qp );
451                         }
452
453                         for( int i = 0; i <= chroma422; i++ )
454                             h->dctf.add8x8_idct_dc( p_dst + 8*i*FDEC_STRIDE, &dct_dc[4*i] );
455                         h->mb.i_cbp_chroma = 1;
456                     }
457                 }
458             }
459             return;
460         }
461     }
462
463     for( int ch = 0; ch < 2; ch++ )
464     {
465         pixel *p_src = h->mb.pic.p_fenc[1+ch];
466         pixel *p_dst = h->mb.pic.p_fdec[1+ch];
467         int i_decimate_score = b_decimate ? 0 : 7;
468         int nz_ac = 0;
469
470         ALIGNED_ARRAY_N( dctcoef, dct4x4,[8],[16] );
471
472         if( h->mb.b_lossless )
473         {
474             static const uint8_t chroma422_scan[8] = { 0, 2, 1, 5, 3, 6, 4, 7 };
475
476             for( int i = 0; i < (chroma422?8:4); i++ )
477             {
478                 int oe = 4*(i&1) + 4*(i>>1)*FENC_STRIDE;
479                 int od = 4*(i&1) + 4*(i>>1)*FDEC_STRIDE;
480                 nz = h->zigzagf.sub_4x4ac( h->dct.luma4x4[16+i+(chroma422?i&4:0)+ch*16], p_src+oe, p_dst+od,
481                                            &h->dct.chroma_dc[ch][chroma422?chroma422_scan[i]:i] );
482                 h->mb.cache.non_zero_count[x264_scan8[16+i+(chroma422?i&4:0)+ch*16]] = nz;
483                 h->mb.i_cbp_chroma |= nz;
484             }
485             h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+ch]] = array_non_zero( h->dct.chroma_dc[ch], chroma422?8:4 );
486             continue;
487         }
488
489         for( int i = 0; i <= chroma422; i++ )
490             h->dctf.sub8x8_dct( &dct4x4[4*i], p_src + 8*i*FENC_STRIDE, p_dst + 8*i*FDEC_STRIDE );
491
492         if( h->mb.b_noise_reduction )
493             for( int i = 0; i < (chroma422?8:4); i++ )
494                 h->quantf.denoise_dct( dct4x4[i], h->nr_residual_sum[2], h->nr_offset[2], 16 );
495
496         if( chroma422 )
497             h->dctf.dct2x4dc( dct_dc, dct4x4 );
498         else
499             dct2x2dc( dct_dc, dct4x4 );
500
501         /* calculate dct coeffs */
502         for( int i8x8 = 0; i8x8 < (chroma422?2:1); i8x8++ )
503         {
504             if( h->mb.b_trellis )
505             {
506                 for( int i4x4 = 0; i4x4 < 4; i4x4++ )
507                 {
508                     if( x264_quant_4x4_trellis( h, dct4x4[i8x8*4+i4x4], CQM_4IC+b_inter, i_qp, DCT_CHROMA_AC, !b_inter, 1, 0 ) )
509                     {
510                         int idx = 16+ch*16+i8x8*8+i4x4;
511                         h->zigzagf.scan_4x4( h->dct.luma4x4[idx], dct4x4[i8x8*4+i4x4] );
512                         h->quantf.dequant_4x4( dct4x4[i8x8*4+i4x4], dequant_mf, i_qp );
513                         if( i_decimate_score < 7 )
514                             i_decimate_score += h->quantf.decimate_score15( h->dct.luma4x4[idx] );
515                         h->mb.cache.non_zero_count[x264_scan8[idx]] = 1;
516                         nz_ac = 1;
517                     }
518                 }
519             }
520             else
521             {
522                 nz = h->quantf.quant_4x4x4( &dct4x4[i8x8*4], h->quant4_mf[CQM_4IC+b_inter][i_qp],
523                                             h->quant4_bias[CQM_4IC+b_inter][i_qp] );
524                 nz_ac |= nz;
525
526                 FOREACH_BIT( i4x4, 0, nz )
527                 {
528                     int idx = 16+ch*16+i8x8*8+i4x4;
529
530                     h->zigzagf.scan_4x4( h->dct.luma4x4[idx], dct4x4[i8x8*4+i4x4] );
531                     h->quantf.dequant_4x4( dct4x4[i8x8*4+i4x4], dequant_mf, i_qp );
532                     if( i_decimate_score < 7 )
533                         i_decimate_score += h->quantf.decimate_score15( h->dct.luma4x4[idx] );
534                     h->mb.cache.non_zero_count[x264_scan8[idx]] = 1;
535                 }
536             }
537         }
538
539         if( h->mb.b_trellis )
540             nz_dc = x264_quant_chroma_dc_trellis( h, dct_dc, i_qp+3*chroma422, !b_inter, CHROMA_DC+ch );
541         else
542         {
543             nz_dc = 0;
544             for( int i = 0; i <= chroma422; i++ )
545                 nz_dc |= h->quantf.quant_2x2_dc( &dct_dc[4*i], h->quant4_mf[CQM_4IC+b_inter][i_qp+3*chroma422][0] >> 1,
546                                                  h->quant4_bias[CQM_4IC+b_inter][i_qp+3*chroma422][0] << 1 );
547         }
548
549         h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+ch]] = nz_dc;
550
551         if( i_decimate_score < 7 || !nz_ac )
552         {
553             /* Decimate the block */
554             M16( &h->mb.cache.non_zero_count[x264_scan8[16+16*ch]] ) = 0;
555             M16( &h->mb.cache.non_zero_count[x264_scan8[18+16*ch]] ) = 0;
556             if( chroma422 )
557             {
558                 M16( &h->mb.cache.non_zero_count[x264_scan8[24+16*ch]] ) = 0;
559                 M16( &h->mb.cache.non_zero_count[x264_scan8[26+16*ch]] ) = 0;
560             }
561
562             if( !nz_dc ) /* Whole block is empty */
563                 continue;
564             if( !x264_mb_optimize_chroma_dc( h, dct_dc, dequant_mf, i_qp+3*chroma422, chroma422 ) )
565             {
566                 h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+ch]] = 0;
567                 continue;
568             }
569             /* DC-only */
570             if( chroma422 )
571             {
572                 zigzag_scan_2x4_dc( h->dct.chroma_dc[ch], dct_dc );
573                 h->quantf.idct_dequant_2x4_dconly( dct_dc, dequant_mf, i_qp+3 );
574             }
575             else
576             {
577                 zigzag_scan_2x2_dc( h->dct.chroma_dc[ch], dct_dc );
578                 idct_dequant_2x2_dconly( dct_dc, dequant_mf, i_qp );
579             }
580
581             for( int i = 0; i <= chroma422; i++ )
582                 h->dctf.add8x8_idct_dc( p_dst + 8*i*FDEC_STRIDE, &dct_dc[4*i] );
583         }
584         else
585         {
586             h->mb.i_cbp_chroma = 1;
587
588             if( nz_dc )
589             {
590                 if( chroma422 )
591                 {
592                     zigzag_scan_2x4_dc( h->dct.chroma_dc[ch], dct_dc );
593                     h->quantf.idct_dequant_2x4_dc( dct_dc, dct4x4, dequant_mf, i_qp+3 );
594                 }
595                 else
596                 {
597                     zigzag_scan_2x2_dc( h->dct.chroma_dc[ch], dct_dc );
598                     idct_dequant_2x2_dc( dct_dc, dct4x4, dequant_mf, i_qp );
599                 }
600             }
601
602             for( int i = 0; i <= chroma422; i++ )
603                 h->dctf.add8x8_idct( p_dst + 8*i*FDEC_STRIDE, &dct4x4[4*i] );
604         }
605     }
606
607     /* 0 = none, 1 = DC only, 2 = DC+AC */
608     h->mb.i_cbp_chroma += (h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+0]] |
609                            h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+1]] | h->mb.i_cbp_chroma);
610 }
611
612 void x264_mb_encode_chroma( x264_t *h, int b_inter, int i_qp )
613 {
614     if( CHROMA_FORMAT == CHROMA_420 )
615         x264_mb_encode_chroma_internal( h, b_inter, i_qp, 0 );
616     else
617         x264_mb_encode_chroma_internal( h, b_inter, i_qp, 1 );
618 }
619
620 static void x264_macroblock_encode_skip( x264_t *h )
621 {
622     M32( &h->mb.cache.non_zero_count[x264_scan8[ 0]] ) = 0;
623     M32( &h->mb.cache.non_zero_count[x264_scan8[ 2]] ) = 0;
624     M32( &h->mb.cache.non_zero_count[x264_scan8[ 8]] ) = 0;
625     M32( &h->mb.cache.non_zero_count[x264_scan8[10]] ) = 0;
626     M32( &h->mb.cache.non_zero_count[x264_scan8[16+ 0]] ) = 0;
627     M32( &h->mb.cache.non_zero_count[x264_scan8[16+ 2]] ) = 0;
628     M32( &h->mb.cache.non_zero_count[x264_scan8[32+ 0]] ) = 0;
629     M32( &h->mb.cache.non_zero_count[x264_scan8[32+ 2]] ) = 0;
630     if( CHROMA_FORMAT >= CHROMA_422 )
631     {
632         M32( &h->mb.cache.non_zero_count[x264_scan8[16+ 8]] ) = 0;
633         M32( &h->mb.cache.non_zero_count[x264_scan8[16+10]] ) = 0;
634         M32( &h->mb.cache.non_zero_count[x264_scan8[32+ 8]] ) = 0;
635         M32( &h->mb.cache.non_zero_count[x264_scan8[32+10]] ) = 0;
636     }
637     h->mb.i_cbp_luma = 0;
638     h->mb.i_cbp_chroma = 0;
639     h->mb.i_cbp_chroma422 = 0;
640     h->mb.cbp[h->mb.i_mb_xy] = 0;
641     x264_reset_mv_predictor_mpeg2( h );
642 }
643
644 /*****************************************************************************
645  * Intra prediction for predictive lossless mode.
646  *****************************************************************************/
647
648 void x264_predict_lossless_chroma( x264_t *h, int i_mode )
649 {
650     int height = 16 >> CHROMA_V_SHIFT;
651     if( i_mode == I_PRED_CHROMA_V )
652     {
653         h->mc.copy[PIXEL_8x8]( h->mb.pic.p_fdec[1], FDEC_STRIDE, h->mb.pic.p_fenc[1]-FENC_STRIDE, FENC_STRIDE, height );
654         h->mc.copy[PIXEL_8x8]( h->mb.pic.p_fdec[2], FDEC_STRIDE, h->mb.pic.p_fenc[2]-FENC_STRIDE, FENC_STRIDE, height );
655         memcpy( h->mb.pic.p_fdec[1], h->mb.pic.p_fdec[1]-FDEC_STRIDE, 8*sizeof(pixel) );
656         memcpy( h->mb.pic.p_fdec[2], h->mb.pic.p_fdec[2]-FDEC_STRIDE, 8*sizeof(pixel) );
657     }
658     else if( i_mode == I_PRED_CHROMA_H )
659     {
660         h->mc.copy[PIXEL_8x8]( h->mb.pic.p_fdec[1], FDEC_STRIDE, h->mb.pic.p_fenc[1]-1, FENC_STRIDE, height );
661         h->mc.copy[PIXEL_8x8]( h->mb.pic.p_fdec[2], FDEC_STRIDE, h->mb.pic.p_fenc[2]-1, FENC_STRIDE, height );
662         x264_copy_column8( h->mb.pic.p_fdec[1]+4*FDEC_STRIDE, h->mb.pic.p_fdec[1]+4*FDEC_STRIDE-1 );
663         x264_copy_column8( h->mb.pic.p_fdec[2]+4*FDEC_STRIDE, h->mb.pic.p_fdec[2]+4*FDEC_STRIDE-1 );
664         if( CHROMA_FORMAT == CHROMA_422 )
665         {
666             x264_copy_column8( h->mb.pic.p_fdec[1]+12*FDEC_STRIDE, h->mb.pic.p_fdec[1]+12*FDEC_STRIDE-1 );
667             x264_copy_column8( h->mb.pic.p_fdec[2]+12*FDEC_STRIDE, h->mb.pic.p_fdec[2]+12*FDEC_STRIDE-1 );
668         }
669     }
670     else
671     {
672         h->predict_chroma[i_mode]( h->mb.pic.p_fdec[1] );
673         h->predict_chroma[i_mode]( h->mb.pic.p_fdec[2] );
674     }
675 }
676
677 void x264_predict_lossless_4x4( x264_t *h, pixel *p_dst, int p, int idx, int i_mode )
678 {
679     int stride = h->fenc->i_stride[p] << MB_INTERLACED;
680     pixel *p_src = h->mb.pic.p_fenc_plane[p] + block_idx_x[idx]*4 + block_idx_y[idx]*4 * stride;
681
682     if( i_mode == I_PRED_4x4_V )
683         h->mc.copy[PIXEL_4x4]( p_dst, FDEC_STRIDE, p_src-stride, stride, 4 );
684     else if( i_mode == I_PRED_4x4_H )
685         h->mc.copy[PIXEL_4x4]( p_dst, FDEC_STRIDE, p_src-1, stride, 4 );
686     else
687         h->predict_4x4[i_mode]( p_dst );
688 }
689
690 void x264_predict_lossless_8x8( x264_t *h, pixel *p_dst, int p, int idx, int i_mode, pixel edge[36] )
691 {
692     int stride = h->fenc->i_stride[p] << MB_INTERLACED;
693     pixel *p_src = h->mb.pic.p_fenc_plane[p] + (idx&1)*8 + (idx>>1)*8*stride;
694
695     if( i_mode == I_PRED_8x8_V )
696         h->mc.copy[PIXEL_8x8]( p_dst, FDEC_STRIDE, p_src-stride, stride, 8 );
697     else if( i_mode == I_PRED_8x8_H )
698         h->mc.copy[PIXEL_8x8]( p_dst, FDEC_STRIDE, p_src-1, stride, 8 );
699     else
700         h->predict_8x8[i_mode]( p_dst, edge );
701 }
702
703 void x264_predict_lossless_16x16( x264_t *h, int p, int i_mode )
704 {
705     int stride = h->fenc->i_stride[p] << MB_INTERLACED;
706     if( i_mode == I_PRED_16x16_V )
707         h->mc.copy[PIXEL_16x16]( h->mb.pic.p_fdec[p], FDEC_STRIDE, h->mb.pic.p_fenc_plane[p]-stride, stride, 16 );
708     else if( i_mode == I_PRED_16x16_H )
709         h->mc.copy_16x16_unaligned( h->mb.pic.p_fdec[p], FDEC_STRIDE, h->mb.pic.p_fenc_plane[p]-1, stride, 16 );
710     else
711         h->predict_16x16[i_mode]( h->mb.pic.p_fdec[p] );
712 }
713
714 /*****************************************************************************
715  * x264_macroblock_encode:
716  *****************************************************************************/
717 static ALWAYS_INLINE void x264_macroblock_encode_internal( x264_t *h, int plane_count, int chroma )
718 {
719     int i_qp = h->mb.i_qp;
720     int b_decimate = h->mb.b_dct_decimate;
721     int b_force_no_skip = 0;
722     int nz;
723     h->mb.i_cbp_luma = 0;
724     for( int p = 0; p < plane_count; p++ )
725         h->mb.cache.non_zero_count[x264_scan8[LUMA_DC+p]] = 0;
726
727     if( h->mb.i_type == I_PCM )
728     {
729         /* if PCM is chosen, we need to store reconstructed frame data */
730         for( int p = 0; p < plane_count; p++ )
731             h->mc.copy[PIXEL_16x16]( h->mb.pic.p_fdec[p], FDEC_STRIDE, h->mb.pic.p_fenc[p], FENC_STRIDE, 16 );
732         if( chroma )
733         {
734             int height = 16 >> CHROMA_V_SHIFT;
735             h->mc.copy[PIXEL_8x8]  ( h->mb.pic.p_fdec[1], FDEC_STRIDE, h->mb.pic.p_fenc[1], FENC_STRIDE, height );
736             h->mc.copy[PIXEL_8x8]  ( h->mb.pic.p_fdec[2], FDEC_STRIDE, h->mb.pic.p_fenc[2], FENC_STRIDE, height );
737         }
738         return;
739     }
740
741     if( !h->mb.b_allow_skip )
742     {
743         b_force_no_skip = 1;
744         if( IS_SKIP(h->mb.i_type) )
745         {
746             if( h->mb.i_type == P_SKIP )
747                 h->mb.i_type = P_L0;
748             else if( h->mb.i_type == B_SKIP )
749                 h->mb.i_type = B_DIRECT;
750         }
751     }
752
753     if( MPEG2 && ( (h->mb.i_mb_x == 0 || h->mb.i_mb_x == h->mb.i_mb_width-1) ||
754                    (h->sh.i_type == SLICE_TYPE_B && IS_INTRA( h->mb.i_mb_type_left[0])) ))
755     {
756         /* First and last macroblock in a slice is not allowed to be a skip
757          * In B-frames a skip cannot follow an intra mb */
758         b_force_no_skip = 1;
759         if( h->mb.i_type == P_SKIP )
760             h->mb.i_type = P_L0;
761         else if( h->mb.i_type == B_SKIP )
762             h->mb.i_type = h->mb.i_mb_type_left[0];
763         /* Do motion compensation if we have a non-zero mv */
764         if( h->mb.i_type == P_L0 )
765         {
766             if( M32( h->mb.cache.mv[0][X264_SCAN8_0] )
767                 | ( MB_INTERLACED && M32( h->mb.cache.mv[0][X264_SCAN8_0+8*2] ) ) )
768                 x264_mb_mc( h );
769         }
770         else if( h->mb.i_type != I_16x16 )
771         {
772             if( ( M32( h->mb.cache.mv[0][X264_SCAN8_0] ) | M32( h->mb.cache.mv[1][X264_SCAN8_0] ) )
773                 | ( MB_INTERLACED && ( M32( h->mb.cache.mv[0][X264_SCAN8_0+8*2] ) | M32( h->mb.cache.mv[1][X264_SCAN8_0+8*2] ) ) ) )
774                 x264_mb_mc( h );
775         }
776     }
777
778     if( h->mb.i_type == P_SKIP )
779     {
780         /* don't do pskip motion compensation if it was already done in macroblock_analyse */
781         if( !h->mb.b_skip_mc )
782         {
783             int mvx = x264_clip3( h->mb.cache.mv[0][x264_scan8[0]][0],
784                                   h->mb.mv_min[0], h->mb.mv_max[0] );
785             int mvy = x264_clip3( h->mb.cache.mv[0][x264_scan8[0]][1],
786                                   h->mb.mv_min[1], h->mb.mv_max[1] );
787
788             for( int p = 0; p < plane_count; p++ )
789                 h->mc.mc_luma( h->mb.pic.p_fdec[p], FDEC_STRIDE,
790                                &h->mb.pic.p_fref[0][0][p*4], h->mb.pic.i_stride[p],
791                                mvx, mvy, 16, 16, &h->sh.weight[0][p] );
792
793             if( chroma )
794             {
795                 int v_shift = CHROMA_V_SHIFT;
796                 int height = 16 >> v_shift;
797
798                 /* Special case for mv0, which is (of course) very common in P-skip mode. */
799                 if( mvx | mvy )
800                     h->mc.mc_chroma( h->mb.pic.p_fdec[1], h->mb.pic.p_fdec[2], FDEC_STRIDE,
801                                      h->mb.pic.p_fref[0][0][4], h->mb.pic.i_stride[1],
802                                      mvx, 2*mvy>>v_shift, 8, height );
803                 else
804                     h->mc.load_deinterleave_chroma_fdec( h->mb.pic.p_fdec[1], h->mb.pic.p_fref[0][0][4],
805                                                          h->mb.pic.i_stride[1], height );
806
807                 if( h->sh.weight[0][1].weightfn )
808                     h->sh.weight[0][1].weightfn[8>>2]( h->mb.pic.p_fdec[1], FDEC_STRIDE,
809                                                        h->mb.pic.p_fdec[1], FDEC_STRIDE,
810                                                        &h->sh.weight[0][1], height );
811                 if( h->sh.weight[0][2].weightfn )
812                     h->sh.weight[0][2].weightfn[8>>2]( h->mb.pic.p_fdec[2], FDEC_STRIDE,
813                                                        h->mb.pic.p_fdec[2], FDEC_STRIDE,
814                                                        &h->sh.weight[0][2], height );
815             }
816         }
817
818         x264_macroblock_encode_skip( h );
819         return;
820     }
821     if( h->mb.i_type == B_SKIP )
822     {
823         /* don't do bskip motion compensation if it was already done in macroblock_analyse */
824         if( !h->mb.b_skip_mc )
825             x264_mb_mc( h );
826         x264_macroblock_encode_skip( h );
827         return;
828     }
829
830     if( h->mb.i_type == I_16x16 )
831     {
832         h->mb.b_transform_8x8 = 0;
833
834         /* encode the 16x16 macroblock */
835         if( MPEG2 )
836         {
837             /* luma */
838             for( int i = 0; i < 4; i++ )
839             {
840                 h->predict_8x8_mpeg2( &h->mb.pic.p_fdec[0][8 * (i&1) + 8 * (i>>1) * FDEC_STRIDE], 0 );
841                 x264_mb_encode_intra_block_mpeg2( h, i, i_qp );
842             }
843             /* chroma */
844             int blockcount = CHROMA_FORMAT == CHROMA_422 ? 4 : 2;
845             for( int i = 0; i < blockcount; i++ )
846             {
847                 h->predict_8x8_mpeg2( &h->mb.pic.p_fdec[1 + (i&1)][8 * (i>>1) * FDEC_STRIDE], 0 );
848                 x264_mb_encode_intra_block_mpeg2( h, i+4, i_qp );
849             }
850             /* reset mvp */
851             if( h->sh.i_type == SLICE_TYPE_P || h->sh.i_type == SLICE_TYPE_B )
852                 x264_reset_mv_predictor_mpeg2( h );
853         }
854         else
855         {
856             for( int p = 0; p < plane_count; p++, i_qp = h->mb.i_chroma_qp )
857                 x264_mb_encode_i16x16( h, p, i_qp );
858         }
859     }
860     else if( h->mb.i_type == I_8x8 )
861     {
862         h->mb.b_transform_8x8 = 1;
863         /* If we already encoded 3 of the 4 i8x8 blocks, we don't have to do them again. */
864         if( h->mb.i_skip_intra )
865         {
866             h->mc.copy[PIXEL_16x16]( h->mb.pic.p_fdec[0], FDEC_STRIDE, h->mb.pic.i8x8_fdec_buf, 16, 16 );
867             M32( &h->mb.cache.non_zero_count[x264_scan8[ 0]] ) = h->mb.pic.i8x8_nnz_buf[0];
868             M32( &h->mb.cache.non_zero_count[x264_scan8[ 2]] ) = h->mb.pic.i8x8_nnz_buf[1];
869             M32( &h->mb.cache.non_zero_count[x264_scan8[ 8]] ) = h->mb.pic.i8x8_nnz_buf[2];
870             M32( &h->mb.cache.non_zero_count[x264_scan8[10]] ) = h->mb.pic.i8x8_nnz_buf[3];
871             h->mb.i_cbp_luma = h->mb.pic.i8x8_cbp;
872             /* In RD mode, restore the now-overwritten DCT data. */
873             if( h->mb.i_skip_intra == 2 )
874                 h->mc.memcpy_aligned( h->dct.luma8x8, h->mb.pic.i8x8_dct_buf, sizeof(h->mb.pic.i8x8_dct_buf) );
875         }
876         for( int p = 0; p < plane_count; p++, i_qp = h->mb.i_chroma_qp )
877         {
878             for( int i = (p == 0 && h->mb.i_skip_intra) ? 3 : 0 ; i < 4; i++ )
879             {
880                 int i_mode = h->mb.cache.intra4x4_pred_mode[x264_scan8[4*i]];
881                 x264_mb_encode_i8x8( h, p, i, i_qp, i_mode, NULL, 1 );
882             }
883         }
884     }
885     else if( h->mb.i_type == I_4x4 )
886     {
887         h->mb.b_transform_8x8 = 0;
888         /* If we already encoded 15 of the 16 i4x4 blocks, we don't have to do them again. */
889         if( h->mb.i_skip_intra )
890         {
891             h->mc.copy[PIXEL_16x16]( h->mb.pic.p_fdec[0], FDEC_STRIDE, h->mb.pic.i4x4_fdec_buf, 16, 16 );
892             M32( &h->mb.cache.non_zero_count[x264_scan8[ 0]] ) = h->mb.pic.i4x4_nnz_buf[0];
893             M32( &h->mb.cache.non_zero_count[x264_scan8[ 2]] ) = h->mb.pic.i4x4_nnz_buf[1];
894             M32( &h->mb.cache.non_zero_count[x264_scan8[ 8]] ) = h->mb.pic.i4x4_nnz_buf[2];
895             M32( &h->mb.cache.non_zero_count[x264_scan8[10]] ) = h->mb.pic.i4x4_nnz_buf[3];
896             h->mb.i_cbp_luma = h->mb.pic.i4x4_cbp;
897             /* In RD mode, restore the now-overwritten DCT data. */
898             if( h->mb.i_skip_intra == 2 )
899                 h->mc.memcpy_aligned( h->dct.luma4x4, h->mb.pic.i4x4_dct_buf, sizeof(h->mb.pic.i4x4_dct_buf) );
900         }
901         for( int p = 0; p < plane_count; p++, i_qp = h->mb.i_chroma_qp )
902         {
903             for( int i = (p == 0 && h->mb.i_skip_intra) ? 15 : 0 ; i < 16; i++ )
904             {
905                 pixel *p_dst = &h->mb.pic.p_fdec[p][block_idx_xy_fdec[i]];
906                 int i_mode = h->mb.cache.intra4x4_pred_mode[x264_scan8[i]];
907
908                 if( (h->mb.i_neighbour4[i] & (MB_TOPRIGHT|MB_TOP)) == MB_TOP )
909                     /* emulate missing topright samples */
910                     MPIXEL_X4( &p_dst[4-FDEC_STRIDE] ) = PIXEL_SPLAT_X4( p_dst[3-FDEC_STRIDE] );
911
912                 x264_mb_encode_i4x4( h, p, i, i_qp, i_mode, 1 );
913             }
914         }
915     }
916     else    /* Inter MB */
917     {
918         int i_decimate_mb = 0;
919
920         /* Don't repeat motion compensation if it was already done in non-RD transform analysis */
921         if( !h->mb.b_skip_mc )
922             x264_mb_mc( h );
923
924         if( MPEG2 )
925         {
926             int blockcount = CHROMA_FORMAT == CHROMA_422 ? 8 : 6;
927             h->mb.i_cbp_chroma = 0;
928             h->mb.i_cbp_chroma422 = 0;
929             for( int i = 0; i < blockcount; i++ )
930                 x264_mb_encode_inter_block_mpeg2( h, i, i_qp );
931         }
932         else if( h->mb.b_lossless )
933         {
934             if( h->mb.b_transform_8x8 )
935                 for( int p = 0; p < plane_count; p++ )
936                     for( int i8x8 = 0; i8x8 < 4; i8x8++ )
937                     {
938                         int x = i8x8&1;
939                         int y = i8x8>>1;
940                         nz = h->zigzagf.sub_8x8( h->dct.luma8x8[p*4+i8x8], h->mb.pic.p_fenc[p] + 8*x + 8*y*FENC_STRIDE,
941                                                                            h->mb.pic.p_fdec[p] + 8*x + 8*y*FDEC_STRIDE );
942                         STORE_8x8_NNZ( p, i8x8, nz );
943                         h->mb.i_cbp_luma |= nz << i8x8;
944                     }
945             else
946                 for( int p = 0; p < plane_count; p++ )
947                     for( int i4x4 = 0; i4x4 < 16; i4x4++ )
948                     {
949                         nz = h->zigzagf.sub_4x4( h->dct.luma4x4[p*16+i4x4],
950                                                  h->mb.pic.p_fenc[p]+block_idx_xy_fenc[i4x4],
951                                                  h->mb.pic.p_fdec[p]+block_idx_xy_fdec[i4x4] );
952                         h->mb.cache.non_zero_count[x264_scan8[p*16+i4x4]] = nz;
953                         h->mb.i_cbp_luma |= nz << (i4x4>>2);
954                     }
955         }
956         else if( h->mb.b_transform_8x8 )
957         {
958             ALIGNED_ARRAY_N( dctcoef, dct8x8,[4],[64] );
959             b_decimate &= !h->mb.b_trellis || !h->param.b_cabac; // 8x8 trellis is inherently optimal decimation for CABAC
960
961             for( int p = 0; p < plane_count; p++, i_qp = h->mb.i_chroma_qp )
962             {
963                 CLEAR_16x16_NNZ( p );
964                 h->dctf.sub16x16_dct8( dct8x8, h->mb.pic.p_fenc[p], h->mb.pic.p_fdec[p] );
965                 h->nr_count[1+!!p*2] += h->mb.b_noise_reduction * 4;
966
967                 int plane_cbp = 0;
968                 for( int idx = 0; idx < 4; idx++ )
969                 {
970                     nz = x264_quant_8x8( h, dct8x8[idx], i_qp, ctx_cat_plane[DCT_LUMA_8x8][p], 0, p, idx );
971
972                     if( nz )
973                     {
974                         h->zigzagf.scan_8x8( h->dct.luma8x8[p*4+idx], dct8x8[idx] );
975                         if( b_decimate )
976                         {
977                             int i_decimate_8x8 = h->quantf.decimate_score64( h->dct.luma8x8[p*4+idx] );
978                             i_decimate_mb += i_decimate_8x8;
979                             if( i_decimate_8x8 >= 4 )
980                                 plane_cbp |= 1<<idx;
981                         }
982                         else
983                             plane_cbp |= 1<<idx;
984                     }
985                 }
986
987                 if( i_decimate_mb >= 6 || !b_decimate )
988                 {
989                     h->mb.i_cbp_luma |= plane_cbp;
990                     FOREACH_BIT( idx, 0, plane_cbp )
991                     {
992                         h->quantf.dequant_8x8( dct8x8[idx], h->dequant8_mf[p?CQM_8PC:CQM_8PY], i_qp );
993                         h->dctf.add8x8_idct8( &h->mb.pic.p_fdec[p][8*(idx&1) + 8*(idx>>1)*FDEC_STRIDE], dct8x8[idx] );
994                         STORE_8x8_NNZ( p, idx, 1 );
995                     }
996                 }
997             }
998         }
999         else
1000         {
1001             ALIGNED_ARRAY_N( dctcoef, dct4x4,[16],[16] );
1002             for( int p = 0; p < plane_count; p++, i_qp = h->mb.i_chroma_qp )
1003             {
1004                 CLEAR_16x16_NNZ( p );
1005                 h->dctf.sub16x16_dct( dct4x4, h->mb.pic.p_fenc[p], h->mb.pic.p_fdec[p] );
1006
1007                 if( h->mb.b_noise_reduction )
1008                 {
1009                     h->nr_count[0+!!p*2] += 16;
1010                     for( int idx = 0; idx < 16; idx++ )
1011                         h->quantf.denoise_dct( dct4x4[idx], h->nr_residual_sum[0+!!p*2], h->nr_offset[0+!!p*2], 16 );
1012                 }
1013
1014                 int plane_cbp = 0;
1015                 for( int i8x8 = 0; i8x8 < 4; i8x8++ )
1016                 {
1017                     int i_decimate_8x8 = b_decimate ? 0 : 6;
1018                     int nnz8x8 = 0;
1019                     if( h->mb.b_trellis )
1020                     {
1021                         for( int i4x4 = 0; i4x4 < 4; i4x4++ )
1022                         {
1023                             int idx = i8x8*4+i4x4;
1024                             if( x264_quant_4x4_trellis( h, dct4x4[idx], CQM_4PY, i_qp, ctx_cat_plane[DCT_LUMA_4x4][p], 0, !!p, p*16+idx ) )
1025                             {
1026                                 h->zigzagf.scan_4x4( h->dct.luma4x4[p*16+idx], dct4x4[idx] );
1027                                 h->quantf.dequant_4x4( dct4x4[idx], h->dequant4_mf[p?CQM_4PC:CQM_4PY], i_qp );
1028                                 if( i_decimate_8x8 < 6 )
1029                                     i_decimate_8x8 += h->quantf.decimate_score16( h->dct.luma4x4[p*16+idx] );
1030                                 h->mb.cache.non_zero_count[x264_scan8[p*16+idx]] = 1;
1031                                 nnz8x8 = 1;
1032                             }
1033                         }
1034                     }
1035                     else
1036                     {
1037                         nnz8x8 = nz = h->quantf.quant_4x4x4( &dct4x4[i8x8*4], h->quant4_mf[CQM_4PY][i_qp], h->quant4_bias[CQM_4PY][i_qp] );
1038                         if( nz )
1039                         {
1040                             FOREACH_BIT( idx, i8x8*4, nz )
1041                             {
1042                                 h->zigzagf.scan_4x4( h->dct.luma4x4[p*16+idx], dct4x4[idx] );
1043                                 h->quantf.dequant_4x4( dct4x4[idx], h->dequant4_mf[p?CQM_4PC:CQM_4PY], i_qp );
1044                                 if( i_decimate_8x8 < 6 )
1045                                     i_decimate_8x8 += h->quantf.decimate_score16( h->dct.luma4x4[p*16+idx] );
1046                                 h->mb.cache.non_zero_count[x264_scan8[p*16+idx]] = 1;
1047                             }
1048                         }
1049                     }
1050                     if( nnz8x8 )
1051                     {
1052                         i_decimate_mb += i_decimate_8x8;
1053                         if( i_decimate_8x8 < 4 )
1054                             STORE_8x8_NNZ( p, i8x8, 0 );
1055                         else
1056                             plane_cbp |= 1<<i8x8;
1057                     }
1058                 }
1059
1060                 if( i_decimate_mb < 6 )
1061                 {
1062                     plane_cbp = 0;
1063                     CLEAR_16x16_NNZ( p );
1064                 }
1065                 else
1066                 {
1067                     h->mb.i_cbp_luma |= plane_cbp;
1068                     FOREACH_BIT( i8x8, 0, plane_cbp )
1069                     {
1070                         h->dctf.add8x8_idct( &h->mb.pic.p_fdec[p][(i8x8&1)*8 + (i8x8>>1)*8*FDEC_STRIDE], &dct4x4[i8x8*4] );
1071                     }
1072                 }
1073             }
1074         }
1075     }
1076
1077     /* encode H.264 chroma */
1078     if( !MPEG2 && chroma )
1079     {
1080         if( IS_INTRA( h->mb.i_type ) )
1081         {
1082             int i_mode = h->mb.i_chroma_pred_mode;
1083             if( h->mb.b_lossless )
1084                 x264_predict_lossless_chroma( h, i_mode );
1085             else
1086             {
1087                 h->predict_chroma[i_mode]( h->mb.pic.p_fdec[1] );
1088                 h->predict_chroma[i_mode]( h->mb.pic.p_fdec[2] );
1089             }
1090         }
1091         /* encode the 8x8 blocks */
1092         x264_mb_encode_chroma( h, !IS_INTRA( h->mb.i_type ), h->mb.i_chroma_qp );
1093     }
1094     else if( !MPEG2 )
1095         h->mb.i_cbp_chroma = 0;
1096
1097     /* store cbp */
1098     int cbp = h->mb.i_cbp_chroma << 4 | h->mb.i_cbp_luma;
1099     if( h->param.b_cabac )
1100         cbp |= h->mb.cache.non_zero_count[x264_scan8[LUMA_DC    ]] << 8
1101             |  h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+0]] << 9
1102             |  h->mb.cache.non_zero_count[x264_scan8[CHROMA_DC+1]] << 10;
1103     h->mb.cbp[h->mb.i_mb_xy] = cbp;
1104
1105     if( MPEG2 && MB_INTERLACED )
1106         return;
1107
1108     /* Check for P_SKIP
1109      * XXX: in the me perhaps we should take x264_mb_predict_mv_pskip into account
1110      *      (if multiple mv give same result)*/
1111     if( !b_force_no_skip )
1112     {
1113         if( MPEG2 && !(h->mb.i_cbp_luma | h->mb.i_cbp_chroma | h->mb.i_cbp_chroma422 ) )
1114         {
1115             if( h->mb.i_type == P_L0 && !M32( h->mb.cache.mv[0][x264_scan8[0]] ) )
1116                 h->mb.i_type = P_SKIP;
1117             else if( h->mb.i_type == h->mb.i_mb_type_left[0] )
1118             {
1119                 if( h->mb.i_type == B_BI_BI &&
1120                     M32( h->mb.mvp[0][0] ) == M32( h->mb.cache.mv[0][x264_scan8[0]] ) &&
1121                     M32( h->mb.mvp[0][1] ) == M32( h->mb.cache.mv[1][x264_scan8[0]] ) )
1122                 {
1123                     h->mb.i_type = B_SKIP;
1124                     h->mb.i_bskip_type = B_BI_BI;
1125                 }
1126                 else if( h->mb.i_type == B_L0_L0 &&
1127                     M32( h->mb.mvp[0][0] ) == M32( h->mb.cache.mv[0][x264_scan8[0]] ) )
1128                 {
1129                     h->mb.i_type = B_SKIP;
1130                     h->mb.i_bskip_type = B_L0_L0;
1131                 }
1132                 else if( h->mb.i_type == B_L1_L1 &&
1133                     M32( h->mb.mvp[0][1] ) == M32( h->mb.cache.mv[1][x264_scan8[0]] ) )
1134                 {
1135                     h->mb.i_type = B_SKIP;
1136                     h->mb.i_bskip_type = B_L1_L1;
1137                 }
1138             }
1139             else if( h->mb.i_mb_type_left[0] == B_SKIP )
1140             {
1141                 if( h->mb.i_type == B_BI_BI && h->mb.i_bskip_type == B_BI_BI &&
1142                     M32( h->mb.mvp[0][0] ) == M32( h->mb.cache.mv[0][x264_scan8[0]] ) &&
1143                     M32( h->mb.mvp[0][1] ) == M32( h->mb.cache.mv[1][x264_scan8[0]] ) )
1144                     h->mb.i_type = B_SKIP;
1145                 else if( h->mb.i_type == B_L0_L0 && h->mb.i_bskip_type == B_L0_L0 &&
1146                     M32( h->mb.mvp[0][0] ) == M32( h->mb.cache.mv[0][x264_scan8[0]] ) )
1147                     h->mb.i_type = B_SKIP;
1148                 else if( h->mb.i_type == B_L1_L1 && h->mb.i_bskip_type == B_L1_L1 &&
1149                     M32( h->mb.mvp[0][1] ) == M32( h->mb.cache.mv[1][x264_scan8[0]] ) )
1150                     h->mb.i_type = B_SKIP;
1151             }
1152         }
1153         else
1154         {
1155             if( h->mb.i_type == P_L0 && h->mb.i_partition == D_16x16 &&
1156                 !(h->mb.i_cbp_luma | h->mb.i_cbp_chroma) &&
1157                 M32( h->mb.cache.mv[0][x264_scan8[0]] ) == M32( h->mb.cache.pskip_mv )
1158                 && h->mb.cache.ref[0][x264_scan8[0]] == 0 )
1159                     h->mb.i_type = P_SKIP;
1160             else if( h->mb.i_type == B_DIRECT && !(h->mb.i_cbp_luma | h->mb.i_cbp_chroma) )
1161                 h->mb.i_type = B_SKIP;
1162         }
1163     }
1164 }
1165
1166 void x264_macroblock_encode( x264_t *h )
1167 {
1168     if( CHROMA444 )
1169         x264_macroblock_encode_internal( h, 3, 0 );
1170     else
1171         x264_macroblock_encode_internal( h, 1, 1 );
1172 }
1173
1174 /*****************************************************************************
1175  * x264_macroblock_probe_skip:
1176  *  Check if the current MB could be encoded as a [PB]_SKIP
1177  *****************************************************************************/
1178 static ALWAYS_INLINE int x264_macroblock_probe_skip_internal( x264_t *h, int b_bidir, int plane_count, int chroma )
1179 {
1180     ALIGNED_ARRAY_N( dctcoef, dct4x4,[8],[16] );
1181     ALIGNED_ARRAY_16( dctcoef, dctscan,[16] );
1182     ALIGNED_4( int16_t mvp[2] );
1183     int i_qp = h->mb.i_qp;
1184
1185     for( int p = 0; p < plane_count; p++, i_qp = h->mb.i_chroma_qp )
1186     {
1187         int quant_cat = p ? CQM_4PC : CQM_4PY;
1188         if( !b_bidir )
1189         {
1190             /* Get the MV */
1191             mvp[0] = x264_clip3( h->mb.cache.pskip_mv[0], h->mb.mv_min[0], h->mb.mv_max[0] );
1192             mvp[1] = x264_clip3( h->mb.cache.pskip_mv[1], h->mb.mv_min[1], h->mb.mv_max[1] );
1193
1194             /* Motion compensation */
1195             h->mc.mc_luma( h->mb.pic.p_fdec[p],    FDEC_STRIDE,
1196                            &h->mb.pic.p_fref[0][0][p*4], h->mb.pic.i_stride[p],
1197                            mvp[0], mvp[1], 16, 16, &h->sh.weight[0][p] );
1198         }
1199
1200         for( int i8x8 = 0, i_decimate_mb = 0; i8x8 < 4; i8x8++ )
1201         {
1202             int fenc_offset = (i8x8&1) * 8 + (i8x8>>1) * FENC_STRIDE * 8;
1203             int fdec_offset = (i8x8&1) * 8 + (i8x8>>1) * FDEC_STRIDE * 8;
1204
1205             h->dctf.sub8x8_dct( dct4x4, h->mb.pic.p_fenc[p] + fenc_offset,
1206                                         h->mb.pic.p_fdec[p] + fdec_offset );
1207
1208             if( h->mb.b_noise_reduction )
1209                 for( int i4x4 = 0; i4x4 < 4; i4x4++ )
1210                     h->quantf.denoise_dct( dct4x4[i4x4], h->nr_residual_sum[0+!!p*2], h->nr_offset[0+!!p*2], 16 );
1211
1212             int nz = h->quantf.quant_4x4x4( dct4x4, h->quant4_mf[quant_cat][i_qp], h->quant4_bias[quant_cat][i_qp] );
1213             FOREACH_BIT( idx, 0, nz )
1214             {
1215                 h->zigzagf.scan_4x4( dctscan, dct4x4[idx] );
1216                 i_decimate_mb += h->quantf.decimate_score16( dctscan );
1217                 if( i_decimate_mb >= 6 )
1218                     return 0;
1219             }
1220         }
1221     }
1222
1223     if( chroma == CHROMA_420 || chroma == CHROMA_422 )
1224     {
1225         i_qp = h->mb.i_chroma_qp;
1226         int chroma422 = chroma == CHROMA_422;
1227         int thresh = chroma422 ? (x264_lambda2_tab[i_qp] + 16) >> 5 : (x264_lambda2_tab[i_qp] + 32) >> 6;
1228         int ssd;
1229         ALIGNED_ARRAY_16( dctcoef, dct_dc,[8] );
1230
1231         if( !b_bidir )
1232         {
1233             /* Special case for mv0, which is (of course) very common in P-skip mode. */
1234             if( M32( mvp ) )
1235                 h->mc.mc_chroma( h->mb.pic.p_fdec[1], h->mb.pic.p_fdec[2], FDEC_STRIDE,
1236                                  h->mb.pic.p_fref[0][0][4], h->mb.pic.i_stride[1],
1237                                  mvp[0], mvp[1]<<chroma422, 8, chroma422?16:8 );
1238             else
1239                 h->mc.load_deinterleave_chroma_fdec( h->mb.pic.p_fdec[1], h->mb.pic.p_fref[0][0][4],
1240                                                      h->mb.pic.i_stride[1], chroma422?16:8 );
1241         }
1242
1243         for( int ch = 0; ch < 2; ch++ )
1244         {
1245             pixel *p_src = h->mb.pic.p_fenc[1+ch];
1246             pixel *p_dst = h->mb.pic.p_fdec[1+ch];
1247
1248             if( !b_bidir && h->sh.weight[0][1+ch].weightfn )
1249                 h->sh.weight[0][1+ch].weightfn[8>>2]( h->mb.pic.p_fdec[1+ch], FDEC_STRIDE,
1250                                                       h->mb.pic.p_fdec[1+ch], FDEC_STRIDE,
1251                                                       &h->sh.weight[0][1+ch], chroma422?16:8 );
1252
1253             /* there is almost never a termination during chroma, but we can't avoid the check entirely */
1254             /* so instead we check SSD and skip the actual check if the score is low enough. */
1255             ssd = h->pixf.ssd[chroma422?PIXEL_8x16:PIXEL_8x8]( p_dst, FDEC_STRIDE, p_src, FENC_STRIDE );
1256             if( ssd < thresh )
1257                 continue;
1258
1259             /* The vast majority of chroma checks will terminate during the DC check or the higher
1260              * threshold check, so we can save time by doing a DC-only DCT. */
1261             if( h->mb.b_noise_reduction )
1262             {
1263                 for( int i = 0; i <= chroma422; i++ )
1264                     h->dctf.sub8x8_dct( &dct4x4[4*i], p_src + 8*i*FENC_STRIDE, p_dst + 8*i*FDEC_STRIDE );
1265
1266                 for( int i4x4 = 0; i4x4 < (chroma422?8:4); i4x4++ )
1267                 {
1268                     h->quantf.denoise_dct( dct4x4[i4x4], h->nr_residual_sum[2], h->nr_offset[2], 16 );
1269                     dct_dc[i4x4] = dct4x4[i4x4][0];
1270                     dct4x4[i4x4][0] = 0;
1271                 }
1272             }
1273             else
1274             {
1275                 if( chroma422 )
1276                     h->dctf.sub8x16_dct_dc( dct_dc, p_src, p_dst );
1277                 else
1278                     h->dctf.sub8x8_dct_dc( dct_dc, p_src, p_dst );
1279             }
1280
1281             for( int i = 0; i <= chroma422; i++ )
1282                 if( h->quantf.quant_2x2_dc( &dct_dc[4*i], h->quant4_mf[CQM_4PC][i_qp+3*chroma422][0] >> 1,
1283                                             h->quant4_bias[CQM_4PC][i_qp+3*chroma422][0] << 1 ) )
1284                     return 0;
1285
1286             /* If there wasn't a termination in DC, we can check against a much higher threshold. */
1287             if( ssd < thresh*4 )
1288                 continue;
1289
1290             if( !h->mb.b_noise_reduction )
1291                 for( int i = 0; i <= chroma422; i++ )
1292                 {
1293                     h->dctf.sub8x8_dct( &dct4x4[4*i], p_src + 8*i*FENC_STRIDE, p_dst + 8*i*FDEC_STRIDE );
1294                     dct4x4[i*4+0][0] = 0;
1295                     dct4x4[i*4+1][0] = 0;
1296                     dct4x4[i*4+2][0] = 0;
1297                     dct4x4[i*4+3][0] = 0;
1298                 }
1299
1300             /* calculate dct coeffs */
1301             for( int i8x8 = 0, i_decimate_mb = 0; i8x8 < (chroma422?2:1); i8x8++ )
1302             {
1303                 int nz = h->quantf.quant_4x4x4( &dct4x4[i8x8*4], h->quant4_mf[CQM_4PC][i_qp], h->quant4_bias[CQM_4PC][i_qp] );
1304                 FOREACH_BIT( idx, i8x8*4, nz )
1305                 {
1306                     h->zigzagf.scan_4x4( dctscan, dct4x4[idx] );
1307                     i_decimate_mb += h->quantf.decimate_score15( dctscan );
1308                     if( i_decimate_mb >= 7 )
1309                         return 0;
1310                 }
1311             }
1312         }
1313     }
1314
1315     h->mb.b_skip_mc = 1;
1316     return 1;
1317 }
1318
1319 int x264_macroblock_probe_skip_mpeg2( x264_t *h, int b_bidir )
1320 {
1321     ALIGNED_ARRAY_16( dctcoef, dct8x8,[8],[64] );
1322     ALIGNED_ARRAY_16( dctcoef, dctscan8,[64] );
1323
1324     int i_qp = h->mb.i_qp;
1325
1326     if( !b_bidir )
1327     {
1328         /* Motion compensation */
1329         h->mc.mc_luma( h->mb.pic.p_fdec[0],    FDEC_STRIDE,
1330                        h->mb.pic.p_fref[0][0], h->mb.pic.i_stride[0],
1331                        0, 0, 16, 16, &h->sh.weight[0][0] );
1332     }
1333
1334     for( int idx = 0, i_decimate_mb = 0; idx < 4; idx++ )
1335     {
1336         int x = idx&1;
1337         int y = idx>>1;
1338         pixel *p_src = &h->mb.pic.p_fenc[0][8*x + 8*y*FENC_STRIDE];
1339         pixel *p_dst = &h->mb.pic.p_fdec[0][8*x + 8*y*FDEC_STRIDE];;
1340
1341         h->dctf.sub8x8_dct8( dct8x8[idx], p_src, p_dst );
1342
1343         if( h->mb.b_noise_reduction)
1344             h->quantf.denoise_dct( dct8x8[idx], h->nr_residual_sum[0], h->nr_offset[0], 64 );
1345
1346         if( !h->quantf.quant_8x8( dct8x8[idx], h->quant8_mf[CQM_8PY][i_qp], h->quant8_bias[CQM_8PY][i_qp] ) )
1347             continue;
1348         h->zigzagf.scan_8x8( dctscan8, dct8x8[idx] );
1349         i_decimate_mb += h->quantf.decimate_score64( dctscan8 );
1350         if( i_decimate_mb >= 6 )
1351             return 0;
1352     }
1353
1354     /* encode chroma */
1355     if( !b_bidir )
1356     {
1357         h->mc.load_deinterleave_chroma_fdec( h->mb.pic.p_fdec[1], h->mb.pic.p_fref[0][0][4],
1358                                              h->mb.pic.i_stride[1], 16 >> h->mb.chroma_v_shift );
1359     }
1360     int blockcount = CHROMA_FORMAT == CHROMA_422 ? 8 : 6;
1361     for( int idx = 4, i_decimate_mb = 0; idx < blockcount; idx++ )
1362     {
1363         int x = idx&1;
1364         pixel *p_src;
1365         pixel *p_dst;
1366         if( idx < 6 )
1367         {
1368             p_src = h->mb.pic.p_fenc[1+x];
1369             p_dst = h->mb.pic.p_fdec[1+x];
1370         }
1371         else // CHROMA_422
1372         {
1373             p_src = &h->mb.pic.p_fenc[1+x][8*FENC_STRIDE];
1374             p_dst = &h->mb.pic.p_fdec[1+x][8*FDEC_STRIDE];
1375         }
1376         h->dctf.sub8x8_dct8( dct8x8[idx], p_src, p_dst );
1377
1378         if( h->mb.b_noise_reduction)
1379             h->quantf.denoise_dct( dct8x8[idx], h->nr_residual_sum[1], h->nr_offset[1], 64 );
1380
1381         if( !h->quantf.quant_8x8( dct8x8[idx], h->quant8_mf[CQM_8PY][i_qp], h->quant8_bias[CQM_8PY][i_qp] ) )
1382             continue;
1383         h->zigzagf.scan_8x8( dctscan8, dct8x8[idx] );
1384         i_decimate_mb += h->quantf.decimate_score64( dctscan8 );
1385         if( i_decimate_mb >= 6 )
1386             return 0;
1387     }
1388
1389     h->mb.b_skip_mc = 1;
1390     return 1;
1391 }
1392
1393 int x264_macroblock_probe_skip( x264_t *h, int b_bidir )
1394 {
1395     if( MPEG2 )
1396         return x264_macroblock_probe_skip_mpeg2( h, b_bidir );
1397     else if( CHROMA_FORMAT == CHROMA_444 )
1398         return x264_macroblock_probe_skip_internal( h, b_bidir, 3, CHROMA_444 );
1399     else if( CHROMA_FORMAT == CHROMA_422 )
1400         return x264_macroblock_probe_skip_internal( h, b_bidir, 1, CHROMA_422 );
1401     else
1402         return x264_macroblock_probe_skip_internal( h, b_bidir, 1, CHROMA_420 );
1403 }
1404
1405 /****************************************************************************
1406  * DCT-domain noise reduction / adaptive deadzone
1407  * from libavcodec
1408  ****************************************************************************/
1409
1410 void x264_noise_reduction_update( x264_t *h )
1411 {
1412     int count = MPEG2 ? 2 : 3 + CHROMA444;
1413     h->nr_offset = h->nr_offset_denoise;
1414     h->nr_residual_sum = h->nr_residual_sum_buf[0];
1415     h->nr_count = h->nr_count_buf[0];
1416     for( int cat = 0; cat < count; cat++ )
1417     {
1418         int dct8x8 = MPEG2 ? 1 : cat&1;
1419         int size = dct8x8 ? 64 : 16;
1420         const uint32_t *weight = MPEG2 ? x264_mpeg2_weight2_tab :
1421                                  dct8x8 ? x264_dct8_weight2_tab : x264_dct4_weight2_tab;
1422
1423         if( h->nr_count[cat] > (dct8x8 ? (1<<16) : (1<<18)) )
1424         {
1425             for( int i = 0; i < size; i++ )
1426                 h->nr_residual_sum[cat][i] >>= 1;
1427             h->nr_count[cat] >>= 1;
1428         }
1429
1430         for( int i = 0; i < size; i++ )
1431             h->nr_offset[cat][i] =
1432                 ((uint64_t)h->param.analyse.i_noise_reduction * h->nr_count[cat]
1433                  + h->nr_residual_sum[cat][i]/2)
1434               / ((uint64_t)h->nr_residual_sum[cat][i] * weight[i]/256 + 1);
1435
1436         /* Don't denoise DC coefficients */
1437         h->nr_offset[cat][0] = 0;
1438     }
1439 }
1440
1441 /*****************************************************************************
1442  * RD only; 4 calls to this do not make up for one macroblock_encode.
1443  * doesn't transform chroma dc.
1444  *****************************************************************************/
1445 static ALWAYS_INLINE void x264_macroblock_encode_p8x8_internal( x264_t *h, int i8, int plane_count, int chroma )
1446 {
1447     int b_decimate = h->mb.b_dct_decimate;
1448     int i_qp = h->mb.i_qp;
1449     int x = i8&1;
1450     int y = i8>>1;
1451     int nz;
1452     int chroma422 = chroma == CHROMA_422;
1453
1454     h->mb.i_cbp_chroma = 0;
1455     h->mb.i_cbp_luma &= ~(1 << i8);
1456
1457     if( !h->mb.b_skip_mc )
1458         x264_mb_mc_8x8( h, i8 );
1459
1460     if( h->mb.b_lossless )
1461     {
1462         for( int p = 0; p < plane_count; p++ )
1463         {
1464             pixel *p_fenc = h->mb.pic.p_fenc[p] + 8*x + 8*y*FENC_STRIDE;
1465             pixel *p_fdec = h->mb.pic.p_fdec[p] + 8*x + 8*y*FDEC_STRIDE;
1466             int nnz8x8 = 0;
1467             if( h->mb.b_transform_8x8 )
1468             {
1469                 nnz8x8 = h->zigzagf.sub_8x8( h->dct.luma8x8[4*p+i8], p_fenc, p_fdec );
1470                 STORE_8x8_NNZ( p, i8, nnz8x8 );
1471             }
1472             else
1473             {
1474                 for( int i4 = i8*4; i4 < i8*4+4; i4++ )
1475                 {
1476                     nz = h->zigzagf.sub_4x4( h->dct.luma4x4[16*p+i4],
1477                                              h->mb.pic.p_fenc[p]+block_idx_xy_fenc[i4],
1478                                              h->mb.pic.p_fdec[p]+block_idx_xy_fdec[i4] );
1479                     h->mb.cache.non_zero_count[x264_scan8[16*p+i4]] = nz;
1480                     nnz8x8 |= nz;
1481                 }
1482             }
1483             h->mb.i_cbp_luma |= nnz8x8 << i8;
1484         }
1485         if( chroma == CHROMA_420 || chroma == CHROMA_422 )
1486         {
1487             for( int ch = 0; ch < 2; ch++ )
1488             {
1489                 dctcoef dc;
1490                 pixel *p_fenc = h->mb.pic.p_fenc[1+ch] + 4*x + (chroma422?8:4)*y*FENC_STRIDE;
1491                 pixel *p_fdec = h->mb.pic.p_fdec[1+ch] + 4*x + (chroma422?8:4)*y*FDEC_STRIDE;
1492
1493                 for( int i4x4 = 0; i4x4 <= chroma422; i4x4++ )
1494                 {
1495                     int offset = chroma422 ? 8*y + 2*i4x4 + x : i8;
1496                     nz = h->zigzagf.sub_4x4ac( h->dct.luma4x4[16+offset+ch*16], p_fenc+4*i4x4*FENC_STRIDE, p_fdec+4*i4x4*FDEC_STRIDE, &dc );
1497                     h->mb.cache.non_zero_count[x264_scan8[16+offset+ch*16]] = nz;
1498                 }
1499             }
1500             h->mb.i_cbp_chroma = 0x02;
1501         }
1502     }
1503     else
1504     {
1505         if( h->mb.b_transform_8x8 )
1506         {
1507             for( int p = 0; p < plane_count; p++, i_qp = h->mb.i_chroma_qp )
1508             {
1509                 int quant_cat = p ? CQM_8PC : CQM_8PY;
1510                 pixel *p_fenc = h->mb.pic.p_fenc[p] + 8*x + 8*y*FENC_STRIDE;
1511                 pixel *p_fdec = h->mb.pic.p_fdec[p] + 8*x + 8*y*FDEC_STRIDE;
1512                 ALIGNED_ARRAY_N( dctcoef, dct8x8,[64] );
1513
1514                 h->dctf.sub8x8_dct8( dct8x8, p_fenc, p_fdec );
1515                 int nnz8x8 = x264_quant_8x8( h, dct8x8, i_qp, ctx_cat_plane[DCT_LUMA_8x8][p], 0, p, i8 );
1516                 if( nnz8x8 )
1517                 {
1518                     h->zigzagf.scan_8x8( h->dct.luma8x8[4*p+i8], dct8x8 );
1519
1520                     if( b_decimate && !h->mb.b_trellis )
1521                         nnz8x8 = 4 <= h->quantf.decimate_score64( h->dct.luma8x8[4*p+i8] );
1522
1523                     if( nnz8x8 )
1524                     {
1525                         h->quantf.dequant_8x8( dct8x8, h->dequant8_mf[quant_cat], i_qp );
1526                         h->dctf.add8x8_idct8( p_fdec, dct8x8 );
1527                         STORE_8x8_NNZ( p, i8, 1 );
1528                         h->mb.i_cbp_luma |= 1 << i8;
1529                     }
1530                     else
1531                         STORE_8x8_NNZ( p, i8, 0 );
1532                 }
1533                 else
1534                     STORE_8x8_NNZ( p, i8, 0 );
1535             }
1536         }
1537         else
1538         {
1539             for( int p = 0; p < plane_count; p++, i_qp = h->mb.i_chroma_qp )
1540             {
1541                 int quant_cat = p ? CQM_4PC : CQM_4PY;
1542                 pixel *p_fenc = h->mb.pic.p_fenc[p] + 8*x + 8*y*FENC_STRIDE;
1543                 pixel *p_fdec = h->mb.pic.p_fdec[p] + 8*x + 8*y*FDEC_STRIDE;
1544                 int i_decimate_8x8 = b_decimate ? 0 : 4;
1545                 ALIGNED_ARRAY_N( dctcoef, dct4x4,[4],[16] );
1546                 int nnz8x8 = 0;
1547
1548                 h->dctf.sub8x8_dct( dct4x4, p_fenc, p_fdec );
1549                 STORE_8x8_NNZ( p, i8, 0 );
1550
1551                 if( h->mb.b_noise_reduction )
1552                     for( int idx = 0; idx < 4; idx++ )
1553                         h->quantf.denoise_dct( dct4x4[idx], h->nr_residual_sum[0+!!p*2], h->nr_offset[0+!!p*2], 16 );
1554
1555                 if( h->mb.b_trellis )
1556                 {
1557                     for( int i4x4 = 0; i4x4 < 4; i4x4++ )
1558                     {
1559                         if( x264_quant_4x4_trellis( h, dct4x4[i4x4], quant_cat, i_qp, ctx_cat_plane[DCT_LUMA_4x4][p], 0, !!p, i8*4+i4x4+p*16 ) )
1560                         {
1561                             h->zigzagf.scan_4x4( h->dct.luma4x4[p*16+i8*4+i4x4], dct4x4[i4x4] );
1562                             h->quantf.dequant_4x4( dct4x4[i4x4], h->dequant4_mf[quant_cat], i_qp );
1563                             if( i_decimate_8x8 < 4 )
1564                                 i_decimate_8x8 += h->quantf.decimate_score16( h->dct.luma4x4[p*16+i8*4+i4x4] );
1565                             h->mb.cache.non_zero_count[x264_scan8[p*16+i8*4+i4x4]] = 1;
1566                             nnz8x8 = 1;
1567                         }
1568                     }
1569                 }
1570                 else
1571                 {
1572                     nnz8x8 = nz = h->quantf.quant_4x4x4( dct4x4, h->quant4_mf[quant_cat][i_qp], h->quant4_bias[quant_cat][i_qp] );
1573                     if( nz )
1574                     {
1575                         FOREACH_BIT( i4x4, 0, nz )
1576                         {
1577                             h->zigzagf.scan_4x4( h->dct.luma4x4[p*16+i8*4+i4x4], dct4x4[i4x4] );
1578                             h->quantf.dequant_4x4( dct4x4[i4x4], h->dequant4_mf[quant_cat], i_qp );
1579                             if( i_decimate_8x8 < 4 )
1580                                 i_decimate_8x8 += h->quantf.decimate_score16( h->dct.luma4x4[p*16+i8*4+i4x4] );
1581                             h->mb.cache.non_zero_count[x264_scan8[p*16+i8*4+i4x4]] = 1;
1582                         }
1583                     }
1584                 }
1585                 if( nnz8x8 )
1586                 {
1587                     /* decimate this 8x8 block */
1588                     if( i_decimate_8x8 < 4 )
1589                         STORE_8x8_NNZ( p, i8, 0 );
1590                     else
1591                     {
1592                         h->dctf.add8x8_idct( p_fdec, dct4x4 );
1593                         h->mb.i_cbp_luma |= 1 << i8;
1594                     }
1595                 }
1596             }
1597         }
1598
1599         if( chroma == CHROMA_420 || chroma == CHROMA_422 )
1600         {
1601             i_qp = h->mb.i_chroma_qp;
1602             for( int ch = 0; ch < 2; ch++ )
1603             {
1604                 ALIGNED_ARRAY_N( dctcoef, dct4x4,[2],[16] );
1605                 pixel *p_fenc = h->mb.pic.p_fenc[1+ch] + 4*x + (chroma422?8:4)*y*FENC_STRIDE;
1606                 pixel *p_fdec = h->mb.pic.p_fdec[1+ch] + 4*x + (chroma422?8:4)*y*FDEC_STRIDE;
1607
1608                 for( int i4x4 = 0; i4x4 <= chroma422; i4x4++ )
1609                 {
1610                     h->dctf.sub4x4_dct( dct4x4[i4x4], p_fenc + 4*i4x4*FENC_STRIDE, p_fdec + 4*i4x4*FDEC_STRIDE );
1611
1612                     if( h->mb.b_noise_reduction )
1613                         h->quantf.denoise_dct( dct4x4[i4x4], h->nr_residual_sum[2], h->nr_offset[2], 16 );
1614                     dct4x4[i4x4][0] = 0;
1615
1616                     if( h->mb.b_trellis )
1617                         nz = x264_quant_4x4_trellis( h, dct4x4[i4x4], CQM_4PC, i_qp, DCT_CHROMA_AC, 0, 1, 0 );
1618                     else
1619                         nz = h->quantf.quant_4x4( dct4x4[i4x4], h->quant4_mf[CQM_4PC][i_qp], h->quant4_bias[CQM_4PC][i_qp] );
1620
1621                     int offset = chroma422 ? ((5*i8) & 0x09) + 2*i4x4 : i8;
1622                     h->mb.cache.non_zero_count[x264_scan8[16+offset+ch*16]] = nz;
1623                     if( nz )
1624                     {
1625                         h->zigzagf.scan_4x4( h->dct.luma4x4[16+offset+ch*16], dct4x4[i4x4] );
1626                         h->quantf.dequant_4x4( dct4x4[i4x4], h->dequant4_mf[CQM_4PC], i_qp );
1627                         h->dctf.add4x4_idct( p_fdec + 4*i4x4*FDEC_STRIDE, dct4x4[i4x4] );
1628                     }
1629                 }
1630             }
1631             h->mb.i_cbp_chroma = 0x02;
1632         }
1633     }
1634 }
1635
1636 void x264_macroblock_encode_p8x8( x264_t *h, int i8 )
1637 {
1638     if( CHROMA444 )
1639         x264_macroblock_encode_p8x8_internal( h, i8, 3, CHROMA_444 );
1640     else if( CHROMA_FORMAT == CHROMA_422 )
1641         x264_macroblock_encode_p8x8_internal( h, i8, 1, CHROMA_422 );
1642     else
1643         x264_macroblock_encode_p8x8_internal( h, i8, 1, CHROMA_420 );
1644 }
1645
1646 /*****************************************************************************
1647  * RD only, luma only (for 4:2:0)
1648  *****************************************************************************/
1649 static ALWAYS_INLINE void x264_macroblock_encode_p4x4_internal( x264_t *h, int i4, int plane_count )
1650 {
1651     int i_qp = h->mb.i_qp;
1652
1653     for( int p = 0; p < plane_count; p++, i_qp = h->mb.i_chroma_qp )
1654     {
1655         int quant_cat = p ? CQM_4PC : CQM_4PY;
1656         pixel *p_fenc = &h->mb.pic.p_fenc[p][block_idx_xy_fenc[i4]];
1657         pixel *p_fdec = &h->mb.pic.p_fdec[p][block_idx_xy_fdec[i4]];
1658         int nz;
1659
1660         /* Don't need motion compensation as this function is only used in qpel-RD, which caches pixel data. */
1661
1662         if( h->mb.b_lossless )
1663         {
1664             nz = h->zigzagf.sub_4x4( h->dct.luma4x4[p*16+i4], p_fenc, p_fdec );
1665             h->mb.cache.non_zero_count[x264_scan8[p*16+i4]] = nz;
1666         }
1667         else
1668         {
1669             ALIGNED_ARRAY_N( dctcoef, dct4x4,[16] );
1670             h->dctf.sub4x4_dct( dct4x4, p_fenc, p_fdec );
1671             nz = x264_quant_4x4( h, dct4x4, i_qp, ctx_cat_plane[DCT_LUMA_4x4][p], 0, p, i4 );
1672             h->mb.cache.non_zero_count[x264_scan8[p*16+i4]] = nz;
1673             if( nz )
1674             {
1675                 h->zigzagf.scan_4x4( h->dct.luma4x4[p*16+i4], dct4x4 );
1676                 h->quantf.dequant_4x4( dct4x4, h->dequant4_mf[quant_cat], i_qp );
1677                 h->dctf.add4x4_idct( p_fdec, dct4x4 );
1678             }
1679         }
1680     }
1681 }
1682
1683 void x264_macroblock_encode_p4x4( x264_t *h, int i8 )
1684 {
1685     if( CHROMA444 )
1686         x264_macroblock_encode_p4x4_internal( h, i8, 3 );
1687     else
1688         x264_macroblock_encode_p4x4_internal( h, i8, 1 );
1689 }