64-bit cabac optimizations
[x262.git] / common / quant.c
1 /*****************************************************************************
2  * quant.c: quantization and level-run
3  *****************************************************************************
4  * Copyright (C) 2005-2013 x264 project
5  *
6  * Authors: Loren Merritt <lorenm@u.washington.edu>
7  *          Jason Garrett-Glaser <darkshikari@gmail.com>
8  *          Christian Heine <sennindemokrit@gmx.net>
9  *          Henrik Gramner <hengar-6@student.ltu.se>
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2 of the License, or
14  * (at your option) any later version.
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19  * GNU General Public License for more details.
20  *
21  * You should have received a copy of the GNU General Public License
22  * along with this program; if not, write to the Free Software
23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
24  *
25  * This program is also available under a commercial proprietary license.
26  * For more information, contact us at licensing@x264.com.
27  *****************************************************************************/
28
29 #include "common.h"
30
31 #if HAVE_MMX
32 #include "x86/quant.h"
33 #endif
34 #if ARCH_PPC
35 #   include "ppc/quant.h"
36 #endif
37 #if ARCH_ARM
38 #   include "arm/quant.h"
39 #endif
40
41 #define QUANT_ONE( coef, mf, f ) \
42 { \
43     if( (coef) > 0 ) \
44         (coef) = (f + (coef)) * (mf) >> 16; \
45     else \
46         (coef) = - ((f - (coef)) * (mf) >> 16); \
47     nz |= (coef); \
48 }
49
50 static int quant_8x8( dctcoef dct[64], udctcoef mf[64], udctcoef bias[64] )
51 {
52     int nz = 0;
53     for( int i = 0; i < 64; i++ )
54         QUANT_ONE( dct[i], mf[i], bias[i] );
55     return !!nz;
56 }
57
58 static int quant_4x4( dctcoef dct[16], udctcoef mf[16], udctcoef bias[16] )
59 {
60     int nz = 0;
61     for( int i = 0; i < 16; i++ )
62         QUANT_ONE( dct[i], mf[i], bias[i] );
63     return !!nz;
64 }
65
66 static int quant_4x4_dc( dctcoef dct[16], int mf, int bias )
67 {
68     int nz = 0;
69     for( int i = 0; i < 16; i++ )
70         QUANT_ONE( dct[i], mf, bias );
71     return !!nz;
72 }
73
74 static int quant_2x2_dc( dctcoef dct[4], int mf, int bias )
75 {
76     int nz = 0;
77     QUANT_ONE( dct[0], mf, bias );
78     QUANT_ONE( dct[1], mf, bias );
79     QUANT_ONE( dct[2], mf, bias );
80     QUANT_ONE( dct[3], mf, bias );
81     return !!nz;
82 }
83
84 #define DEQUANT_SHL( x ) \
85     dct[x] = ( dct[x] * dequant_mf[i_mf][x] ) << i_qbits
86
87 #define DEQUANT_SHR( x ) \
88     dct[x] = ( dct[x] * dequant_mf[i_mf][x] + f ) >> (-i_qbits)
89
90 static void dequant_4x4( dctcoef dct[16], int dequant_mf[6][16], int i_qp )
91 {
92     const int i_mf = i_qp%6;
93     const int i_qbits = i_qp/6 - 4;
94
95     if( i_qbits >= 0 )
96     {
97         for( int i = 0; i < 16; i++ )
98             DEQUANT_SHL( i );
99     }
100     else
101     {
102         const int f = 1 << (-i_qbits-1);
103         for( int i = 0; i < 16; i++ )
104             DEQUANT_SHR( i );
105     }
106 }
107
108 static void dequant_8x8( dctcoef dct[64], int dequant_mf[6][64], int i_qp )
109 {
110     const int i_mf = i_qp%6;
111     const int i_qbits = i_qp/6 - 6;
112
113     if( i_qbits >= 0 )
114     {
115         for( int i = 0; i < 64; i++ )
116             DEQUANT_SHL( i );
117     }
118     else
119     {
120         const int f = 1 << (-i_qbits-1);
121         for( int i = 0; i < 64; i++ )
122             DEQUANT_SHR( i );
123     }
124 }
125
126 static void dequant_4x4_dc( dctcoef dct[16], int dequant_mf[6][16], int i_qp )
127 {
128     const int i_qbits = i_qp/6 - 6;
129
130     if( i_qbits >= 0 )
131     {
132         const int i_dmf = dequant_mf[i_qp%6][0] << i_qbits;
133         for( int i = 0; i < 16; i++ )
134             dct[i] *= i_dmf;
135     }
136     else
137     {
138         const int i_dmf = dequant_mf[i_qp%6][0];
139         const int f = 1 << (-i_qbits-1);
140         for( int i = 0; i < 16; i++ )
141             dct[i] = ( dct[i] * i_dmf + f ) >> (-i_qbits);
142     }
143 }
144
145 #define IDCT_DEQUANT_2X4_START \
146     int a0 = dct[0] + dct[1]; \
147     int a1 = dct[2] + dct[3]; \
148     int a2 = dct[4] + dct[5]; \
149     int a3 = dct[6] + dct[7]; \
150     int a4 = dct[0] - dct[1]; \
151     int a5 = dct[2] - dct[3]; \
152     int a6 = dct[4] - dct[5]; \
153     int a7 = dct[6] - dct[7]; \
154     int b0 = a0 + a1; \
155     int b1 = a2 + a3; \
156     int b2 = a4 + a5; \
157     int b3 = a6 + a7; \
158     int b4 = a0 - a1; \
159     int b5 = a2 - a3; \
160     int b6 = a4 - a5; \
161     int b7 = a6 - a7;
162
163 static void idct_dequant_2x4_dc( dctcoef dct[8], dctcoef dct4x4[8][16], int dequant_mf[6][16], int i_qp )
164 {
165     IDCT_DEQUANT_2X4_START
166     int dmf = dequant_mf[i_qp%6][0] << i_qp/6;
167     dct4x4[0][0] = ((b0 + b1) * dmf + 32) >> 6;
168     dct4x4[1][0] = ((b2 + b3) * dmf + 32) >> 6;
169     dct4x4[2][0] = ((b0 - b1) * dmf + 32) >> 6;
170     dct4x4[3][0] = ((b2 - b3) * dmf + 32) >> 6;
171     dct4x4[4][0] = ((b4 - b5) * dmf + 32) >> 6;
172     dct4x4[5][0] = ((b6 - b7) * dmf + 32) >> 6;
173     dct4x4[6][0] = ((b4 + b5) * dmf + 32) >> 6;
174     dct4x4[7][0] = ((b6 + b7) * dmf + 32) >> 6;
175 }
176
177 static void idct_dequant_2x4_dconly( dctcoef dct[8], int dequant_mf[6][16], int i_qp )
178 {
179     IDCT_DEQUANT_2X4_START
180     int dmf = dequant_mf[i_qp%6][0] << i_qp/6;
181     dct[0] = ((b0 + b1) * dmf + 32) >> 6;
182     dct[1] = ((b2 + b3) * dmf + 32) >> 6;
183     dct[2] = ((b0 - b1) * dmf + 32) >> 6;
184     dct[3] = ((b2 - b3) * dmf + 32) >> 6;
185     dct[4] = ((b4 - b5) * dmf + 32) >> 6;
186     dct[5] = ((b6 - b7) * dmf + 32) >> 6;
187     dct[6] = ((b4 + b5) * dmf + 32) >> 6;
188     dct[7] = ((b6 + b7) * dmf + 32) >> 6;
189 }
190
191 static ALWAYS_INLINE void optimize_chroma_idct_dequant_2x4( dctcoef out[8], dctcoef dct[8], int dmf )
192 {
193     IDCT_DEQUANT_2X4_START
194     out[0] = ((b0 + b1) * dmf + 2080) >> 6; /* 2080 = 32 + (32<<6) */
195     out[1] = ((b2 + b3) * dmf + 2080) >> 6;
196     out[2] = ((b0 - b1) * dmf + 2080) >> 6;
197     out[3] = ((b2 - b3) * dmf + 2080) >> 6;
198     out[4] = ((b4 - b5) * dmf + 2080) >> 6;
199     out[5] = ((b6 - b7) * dmf + 2080) >> 6;
200     out[6] = ((b4 + b5) * dmf + 2080) >> 6;
201     out[7] = ((b6 + b7) * dmf + 2080) >> 6;
202 }
203 #undef IDCT_DEQUANT_2X4_START
204
205 static ALWAYS_INLINE void optimize_chroma_idct_dequant_2x2( dctcoef out[4], dctcoef dct[4], int dmf )
206 {
207     int d0 = dct[0] + dct[1];
208     int d1 = dct[2] + dct[3];
209     int d2 = dct[0] - dct[1];
210     int d3 = dct[2] - dct[3];
211     out[0] = ((d0 + d1) * dmf >> 5) + 32;
212     out[1] = ((d0 - d1) * dmf >> 5) + 32;
213     out[2] = ((d2 + d3) * dmf >> 5) + 32;
214     out[3] = ((d2 - d3) * dmf >> 5) + 32;
215 }
216
217 static ALWAYS_INLINE int optimize_chroma_round( dctcoef *ref, dctcoef *dct, int dequant_mf, int chroma422 )
218 {
219     dctcoef out[8];
220
221     if( chroma422 )
222         optimize_chroma_idct_dequant_2x4( out, dct, dequant_mf );
223     else
224         optimize_chroma_idct_dequant_2x2( out, dct, dequant_mf );
225
226     int sum = 0;
227     for( int i = 0; i < (chroma422?8:4); i++ )
228         sum |= ref[i] ^ out[i];
229     return sum >> 6;
230 }
231
232 static ALWAYS_INLINE int optimize_chroma_dc_internal( dctcoef *dct, int dequant_mf, int chroma422 )
233 {
234     /* dequant_mf = h->dequant4_mf[CQM_4IC + b_inter][i_qp%6][0] << i_qp/6, max 32*64 */
235     dctcoef dct_orig[8];
236     int coeff, nz;
237
238     if( chroma422 )
239         optimize_chroma_idct_dequant_2x4( dct_orig, dct, dequant_mf );
240     else
241         optimize_chroma_idct_dequant_2x2( dct_orig, dct, dequant_mf );
242
243     /* If the DC coefficients already round to zero, terminate early. */
244     int sum = 0;
245     for( int i = 0; i < (chroma422?8:4); i++ )
246         sum |= dct_orig[i];
247     if( !(sum >> 6) )
248         return 0;
249
250     /* Start with the highest frequency coefficient... is this the best option? */
251     for( nz = 0, coeff = (chroma422?7:3); coeff >= 0; coeff-- )
252     {
253         int level = dct[coeff];
254         int sign = level>>31 | 1; /* dct[coeff] < 0 ? -1 : 1 */
255
256         while( level )
257         {
258             dct[coeff] = level - sign;
259             if( optimize_chroma_round( dct_orig, dct, dequant_mf, chroma422 ) )
260             {
261                 nz = 1;
262                 dct[coeff] = level;
263                 break;
264             }
265             level -= sign;
266         }
267     }
268
269     return nz;
270 }
271
272 static int optimize_chroma_2x2_dc( dctcoef dct[4], int dequant_mf )
273 {
274     return optimize_chroma_dc_internal( dct, dequant_mf, 0 );
275 }
276
277 static int optimize_chroma_2x4_dc( dctcoef dct[8], int dequant_mf )
278 {
279     return optimize_chroma_dc_internal( dct, dequant_mf, 1 );
280 }
281
282 static void x264_denoise_dct( dctcoef *dct, uint32_t *sum, udctcoef *offset, int size )
283 {
284     for( int i = 0; i < size; i++ )
285     {
286         int level = dct[i];
287         int sign = level>>31;
288         level = (level+sign)^sign;
289         sum[i] += level;
290         level -= offset[i];
291         dct[i] = level<0 ? 0 : (level^sign)-sign;
292     }
293 }
294
295 /* (ref: JVT-B118)
296  * x264_mb_decimate_score: given dct coeffs it returns a score to see if we could empty this dct coeffs
297  * to 0 (low score means set it to null)
298  * Used in inter macroblock (luma and chroma)
299  *  luma: for a 8x8 block: if score < 4 -> null
300  *        for the complete mb: if score < 6 -> null
301  *  chroma: for the complete mb: if score < 7 -> null
302  */
303
304 const uint8_t x264_decimate_table4[16] =
305 {
306     3,2,2,1,1,1,0,0,0,0,0,0,0,0,0,0
307 };
308 const uint8_t x264_decimate_table8[64] =
309 {
310     3,3,3,3,2,2,2,2,2,2,2,2,1,1,1,1,
311     1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,
312     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
313     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
314 };
315
316 static int ALWAYS_INLINE x264_decimate_score_internal( dctcoef *dct, int i_max )
317 {
318     const uint8_t *ds_table = (i_max == 64) ? x264_decimate_table8 : x264_decimate_table4;
319     int i_score = 0;
320     int idx = i_max - 1;
321
322     while( idx >= 0 && dct[idx] == 0 )
323         idx--;
324     while( idx >= 0 )
325     {
326         int i_run;
327
328         if( (unsigned)(dct[idx--] + 1) > 2 )
329             return 9;
330
331         i_run = 0;
332         while( idx >= 0 && dct[idx] == 0 )
333         {
334             idx--;
335             i_run++;
336         }
337         i_score += ds_table[i_run];
338     }
339
340     return i_score;
341 }
342
343 static int x264_decimate_score15( dctcoef *dct )
344 {
345     return x264_decimate_score_internal( dct+1, 15 );
346 }
347 static int x264_decimate_score16( dctcoef *dct )
348 {
349     return x264_decimate_score_internal( dct, 16 );
350 }
351 static int x264_decimate_score64( dctcoef *dct )
352 {
353     return x264_decimate_score_internal( dct, 64 );
354 }
355
356 #define last(num)\
357 static int x264_coeff_last##num( dctcoef *l )\
358 {\
359     int i_last = num-1;\
360     while( i_last >= 0 && l[i_last] == 0 )\
361         i_last--;\
362     return i_last;\
363 }
364
365 last(4)
366 last(8)
367 last(15)
368 last(16)
369 last(64)
370
371 #define level_run(num)\
372 static int x264_coeff_level_run##num( dctcoef *dct, x264_run_level_t *runlevel )\
373 {\
374     int i_last = runlevel->last = x264_coeff_last##num(dct);\
375     int i_total = 0;\
376     int mask = 0;\
377     do\
378     {\
379         runlevel->level[i_total++] = dct[i_last];\
380         mask |= 1 << (i_last);\
381         while( --i_last >= 0 && dct[i_last] == 0 );\
382     } while( i_last >= 0 );\
383     runlevel->mask = mask;\
384     return i_total;\
385 }
386
387 level_run(4)
388 level_run(8)
389 level_run(15)
390 level_run(16)
391
392 #if ARCH_X86_64
393 #define INIT_TRELLIS(cpu)\
394     pf->trellis_cabac_4x4 = x264_trellis_cabac_4x4_##cpu;\
395     pf->trellis_cabac_8x8 = x264_trellis_cabac_8x8_##cpu;\
396     pf->trellis_cabac_4x4_psy = x264_trellis_cabac_4x4_psy_##cpu;\
397     pf->trellis_cabac_8x8_psy = x264_trellis_cabac_8x8_psy_##cpu;\
398     pf->trellis_cabac_dc = x264_trellis_cabac_dc_##cpu;\
399     pf->trellis_cabac_chroma_422_dc = x264_trellis_cabac_chroma_422_dc_##cpu;
400 #else
401 #define INIT_TRELLIS(...)
402 #endif
403
404 void x264_quant_init( x264_t *h, int cpu, x264_quant_function_t *pf )
405 {
406     pf->quant_8x8 = quant_8x8;
407     pf->quant_4x4 = quant_4x4;
408     pf->quant_4x4_dc = quant_4x4_dc;
409     pf->quant_2x2_dc = quant_2x2_dc;
410
411     pf->dequant_4x4 = dequant_4x4;
412     pf->dequant_4x4_dc = dequant_4x4_dc;
413     pf->dequant_8x8 = dequant_8x8;
414
415     pf->idct_dequant_2x4_dc = idct_dequant_2x4_dc;
416     pf->idct_dequant_2x4_dconly = idct_dequant_2x4_dconly;
417
418     pf->optimize_chroma_2x2_dc = optimize_chroma_2x2_dc;
419     pf->optimize_chroma_2x4_dc = optimize_chroma_2x4_dc;
420
421     pf->denoise_dct = x264_denoise_dct;
422     pf->decimate_score15 = x264_decimate_score15;
423     pf->decimate_score16 = x264_decimate_score16;
424     pf->decimate_score64 = x264_decimate_score64;
425
426     pf->coeff_last4 = x264_coeff_last4;
427     pf->coeff_last8 = x264_coeff_last8;
428     pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15;
429     pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16;
430     pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64;
431     pf->coeff_level_run4 = x264_coeff_level_run4;
432     pf->coeff_level_run8 = x264_coeff_level_run8;
433     pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15;
434     pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16;
435
436 #if HIGH_BIT_DEPTH
437 #if HAVE_MMX
438     INIT_TRELLIS( sse2 );
439     if( cpu&X264_CPU_MMX2 )
440     {
441 #if ARCH_X86
442         pf->denoise_dct = x264_denoise_dct_mmx;
443         pf->decimate_score15 = x264_decimate_score15_mmx2;
444         pf->decimate_score16 = x264_decimate_score16_mmx2;
445         if( cpu&X264_CPU_SLOW_CTZ )
446         {
447             pf->decimate_score15 = x264_decimate_score15_mmx2_slowctz;
448             pf->decimate_score16 = x264_decimate_score16_mmx2_slowctz;
449         }
450         pf->decimate_score64 = x264_decimate_score64_mmx2;
451         pf->coeff_last8 = x264_coeff_last8_mmx2;
452         pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15_mmx2;
453         pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16_mmx2;
454         pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64_mmx2;
455         pf->coeff_level_run8 = x264_coeff_level_run8_mmx2;
456         pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15_mmx2;
457         pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16_mmx2;
458 #endif
459         pf->coeff_last4 = x264_coeff_last4_mmx2;
460         pf->coeff_level_run4 = x264_coeff_level_run4_mmx2;
461         if( cpu&X264_CPU_LZCNT )
462             pf->coeff_level_run4 = x264_coeff_level_run4_mmx2_lzcnt;
463     }
464     if( cpu&X264_CPU_SSE2 )
465     {
466         pf->quant_4x4 = x264_quant_4x4_sse2;
467         pf->quant_8x8 = x264_quant_8x8_sse2;
468         pf->quant_2x2_dc = x264_quant_2x2_dc_sse2;
469         pf->quant_4x4_dc = x264_quant_4x4_dc_sse2;
470         pf->dequant_4x4 = x264_dequant_4x4_sse2;
471         pf->dequant_8x8 = x264_dequant_8x8_sse2;
472         pf->dequant_4x4_dc = x264_dequant_4x4dc_sse2;
473         pf->denoise_dct = x264_denoise_dct_sse2;
474         pf->decimate_score15 = x264_decimate_score15_sse2;
475         pf->decimate_score16 = x264_decimate_score16_sse2;
476         pf->decimate_score64 = x264_decimate_score64_sse2;
477         if( cpu&X264_CPU_SLOW_CTZ )
478         {
479             pf->decimate_score15 = x264_decimate_score15_sse2_slowctz;
480             pf->decimate_score16 = x264_decimate_score16_sse2_slowctz;
481         }
482         pf->coeff_last8 = x264_coeff_last8_sse2;
483         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2;
484         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2;
485         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2;
486         pf->coeff_level_run8 = x264_coeff_level_run8_sse2;
487         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2;
488         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2;
489         if( cpu&X264_CPU_LZCNT )
490         {
491             pf->coeff_last4 = x264_coeff_last4_mmx2_lzcnt;
492             pf->coeff_last8 = x264_coeff_last8_sse2_lzcnt;
493             pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2_lzcnt;
494             pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2_lzcnt;
495             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2_lzcnt;
496             pf->coeff_level_run8 = x264_coeff_level_run8_sse2_lzcnt;
497             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2_lzcnt;
498             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2_lzcnt;
499         }
500     }
501     if( cpu&X264_CPU_SSSE3 )
502     {
503         pf->quant_4x4 = x264_quant_4x4_ssse3;
504         pf->quant_8x8 = x264_quant_8x8_ssse3;
505         pf->quant_2x2_dc = x264_quant_2x2_dc_ssse3;
506         pf->quant_4x4_dc = x264_quant_4x4_dc_ssse3;
507         pf->denoise_dct = x264_denoise_dct_ssse3;
508         pf->decimate_score15 = x264_decimate_score15_ssse3;
509         pf->decimate_score16 = x264_decimate_score16_ssse3;
510         if( cpu&X264_CPU_SLOW_CTZ )
511         {
512             pf->decimate_score15 = x264_decimate_score15_ssse3_slowctz;
513             pf->decimate_score16 = x264_decimate_score16_ssse3_slowctz;
514         }
515         pf->decimate_score64 = x264_decimate_score64_ssse3;
516         INIT_TRELLIS( ssse3 );
517     }
518     if( cpu&X264_CPU_SSE4 )
519     {
520         pf->quant_2x2_dc = x264_quant_2x2_dc_sse4;
521         pf->quant_4x4_dc = x264_quant_4x4_dc_sse4;
522         pf->quant_4x4 = x264_quant_4x4_sse4;
523         pf->quant_8x8 = x264_quant_8x8_sse4;
524     }
525     if( cpu&X264_CPU_AVX )
526     {
527         pf->denoise_dct = x264_denoise_dct_avx;
528     }
529     if( cpu&X264_CPU_XOP )
530     {
531         pf->dequant_4x4_dc = x264_dequant_4x4dc_xop;
532         if( h->param.i_cqm_preset != X264_CQM_FLAT )
533         {
534             pf->dequant_4x4 = x264_dequant_4x4_xop;
535             pf->dequant_8x8 = x264_dequant_8x8_xop;
536         }
537     }
538 #endif // HAVE_MMX
539 #else // !HIGH_BIT_DEPTH
540 #if HAVE_MMX
541     INIT_TRELLIS( sse2 );
542     if( cpu&X264_CPU_MMX )
543     {
544 #if ARCH_X86
545         pf->quant_4x4 = x264_quant_4x4_mmx;
546         pf->quant_8x8 = x264_quant_8x8_mmx;
547         pf->dequant_4x4 = x264_dequant_4x4_mmx;
548         pf->dequant_4x4_dc = x264_dequant_4x4dc_mmx2;
549         pf->dequant_8x8 = x264_dequant_8x8_mmx;
550         if( h->param.i_cqm_preset == X264_CQM_FLAT )
551         {
552             pf->dequant_4x4 = x264_dequant_4x4_flat16_mmx;
553             pf->dequant_8x8 = x264_dequant_8x8_flat16_mmx;
554         }
555         pf->denoise_dct = x264_denoise_dct_mmx;
556 #endif
557     }
558
559     if( cpu&X264_CPU_MMX2 )
560     {
561         pf->quant_2x2_dc = x264_quant_2x2_dc_mmx2;
562 #if ARCH_X86
563         pf->quant_4x4_dc = x264_quant_4x4_dc_mmx2;
564         pf->decimate_score15 = x264_decimate_score15_mmx2;
565         pf->decimate_score16 = x264_decimate_score16_mmx2;
566         if( cpu&X264_CPU_SLOW_CTZ )
567         {
568             pf->decimate_score15 = x264_decimate_score15_mmx2_slowctz;
569             pf->decimate_score16 = x264_decimate_score16_mmx2_slowctz;
570         }
571         pf->decimate_score64 = x264_decimate_score64_mmx2;
572         pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15_mmx2;
573         pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16_mmx2;
574         pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64_mmx2;
575         pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15_mmx2;
576         pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16_mmx2;
577 #endif
578         pf->coeff_last4 = x264_coeff_last4_mmx2;
579         pf->coeff_last8 = x264_coeff_last8_mmx2;
580         pf->coeff_level_run4 = x264_coeff_level_run4_mmx2;
581         pf->coeff_level_run8 = x264_coeff_level_run8_mmx2;
582         if( cpu&X264_CPU_LZCNT )
583         {
584             pf->coeff_last4 = x264_coeff_last4_mmx2_lzcnt;
585             pf->coeff_last8 = x264_coeff_last8_mmx2_lzcnt;
586             pf->coeff_level_run4 = x264_coeff_level_run4_mmx2_lzcnt;
587             pf->coeff_level_run8 = x264_coeff_level_run8_mmx2_lzcnt;
588         }
589     }
590
591     if( cpu&X264_CPU_SSE2 )
592     {
593         pf->quant_4x4_dc = x264_quant_4x4_dc_sse2;
594         pf->quant_4x4 = x264_quant_4x4_sse2;
595         pf->quant_8x8 = x264_quant_8x8_sse2;
596         pf->dequant_4x4 = x264_dequant_4x4_sse2;
597         pf->dequant_4x4_dc = x264_dequant_4x4dc_sse2;
598         pf->dequant_8x8 = x264_dequant_8x8_sse2;
599         if( h->param.i_cqm_preset == X264_CQM_FLAT )
600         {
601             pf->dequant_4x4 = x264_dequant_4x4_flat16_sse2;
602             pf->dequant_8x8 = x264_dequant_8x8_flat16_sse2;
603         }
604         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_sse2;
605         pf->denoise_dct = x264_denoise_dct_sse2;
606         pf->decimate_score15 = x264_decimate_score15_sse2;
607         pf->decimate_score16 = x264_decimate_score16_sse2;
608         pf->decimate_score64 = x264_decimate_score64_sse2;
609         if( cpu&X264_CPU_SLOW_CTZ )
610         {
611             pf->decimate_score15 = x264_decimate_score15_sse2_slowctz;
612             pf->decimate_score16 = x264_decimate_score16_sse2_slowctz;
613         }
614         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2;
615         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2;
616         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2;
617         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2;
618         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2;
619         if( cpu&X264_CPU_LZCNT )
620         {
621             pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2_lzcnt;
622             pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2_lzcnt;
623             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2_lzcnt;
624             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2_lzcnt;
625             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2_lzcnt;
626         }
627     }
628
629     if( cpu&X264_CPU_SSSE3 )
630     {
631         pf->quant_2x2_dc = x264_quant_2x2_dc_ssse3;
632         pf->quant_4x4_dc = x264_quant_4x4_dc_ssse3;
633         pf->quant_4x4 = x264_quant_4x4_ssse3;
634         pf->quant_8x8 = x264_quant_8x8_ssse3;
635         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_ssse3;
636         pf->denoise_dct = x264_denoise_dct_ssse3;
637         pf->decimate_score15 = x264_decimate_score15_ssse3;
638         pf->decimate_score16 = x264_decimate_score16_ssse3;
639         if( cpu&X264_CPU_SLOW_CTZ )
640         {
641             pf->decimate_score15 = x264_decimate_score15_ssse3_slowctz;
642             pf->decimate_score16 = x264_decimate_score16_ssse3_slowctz;
643         }
644         pf->decimate_score64 = x264_decimate_score64_ssse3;
645         INIT_TRELLIS( ssse3 );
646     }
647
648     if( cpu&X264_CPU_SSE4 )
649     {
650         pf->quant_4x4_dc = x264_quant_4x4_dc_sse4;
651         pf->quant_4x4 = x264_quant_4x4_sse4;
652         pf->quant_8x8 = x264_quant_8x8_sse4;
653         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_sse4;
654     }
655
656     if( cpu&X264_CPU_AVX )
657     {
658         pf->dequant_4x4_dc = x264_dequant_4x4dc_avx;
659         if( h->param.i_cqm_preset != X264_CQM_FLAT )
660         {
661             pf->dequant_4x4 = x264_dequant_4x4_avx;
662             pf->dequant_8x8 = x264_dequant_8x8_avx;
663         }
664         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_avx;
665         pf->denoise_dct = x264_denoise_dct_avx;
666     }
667
668     if( cpu&X264_CPU_XOP )
669     {
670         if( h->param.i_cqm_preset != X264_CQM_FLAT )
671         {
672             pf->dequant_4x4 = x264_dequant_4x4_xop;
673             pf->dequant_8x8 = x264_dequant_8x8_xop;
674         }
675     }
676 #endif // HAVE_MMX
677
678 #if HAVE_ALTIVEC
679     if( cpu&X264_CPU_ALTIVEC ) {
680         pf->quant_2x2_dc = x264_quant_2x2_dc_altivec;
681         pf->quant_4x4_dc = x264_quant_4x4_dc_altivec;
682         pf->quant_4x4 = x264_quant_4x4_altivec;
683         pf->quant_8x8 = x264_quant_8x8_altivec;
684
685         pf->dequant_4x4 = x264_dequant_4x4_altivec;
686         pf->dequant_8x8 = x264_dequant_8x8_altivec;
687     }
688 #endif
689
690 #if HAVE_ARMV6
691     if( cpu&X264_CPU_ARMV6 )
692         pf->coeff_last4 = x264_coeff_last4_arm;
693
694     if( cpu&X264_CPU_NEON )
695     {
696         pf->quant_2x2_dc   = x264_quant_2x2_dc_neon;
697         pf->quant_4x4      = x264_quant_4x4_neon;
698         pf->quant_4x4_dc   = x264_quant_4x4_dc_neon;
699         pf->quant_8x8      = x264_quant_8x8_neon;
700         pf->dequant_4x4    = x264_dequant_4x4_neon;
701         pf->dequant_4x4_dc = x264_dequant_4x4_dc_neon;
702         pf->dequant_8x8    = x264_dequant_8x8_neon;
703         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_neon;
704         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_neon;
705         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_neon;
706     }
707 #endif
708 #endif // HIGH_BIT_DEPTH
709     pf->coeff_last[DCT_LUMA_DC]     = pf->coeff_last[DCT_CHROMAU_DC]  = pf->coeff_last[DCT_CHROMAV_DC] =
710     pf->coeff_last[DCT_CHROMAU_4x4] = pf->coeff_last[DCT_CHROMAV_4x4] = pf->coeff_last[DCT_LUMA_4x4];
711     pf->coeff_last[DCT_CHROMA_AC]   = pf->coeff_last[DCT_CHROMAU_AC]  =
712     pf->coeff_last[DCT_CHROMAV_AC]  = pf->coeff_last[DCT_LUMA_AC];
713     pf->coeff_last[DCT_CHROMAU_8x8] = pf->coeff_last[DCT_CHROMAV_8x8] = pf->coeff_last[DCT_LUMA_8x8];
714
715     pf->coeff_level_run[DCT_LUMA_DC]     = pf->coeff_level_run[DCT_CHROMAU_DC]  = pf->coeff_level_run[DCT_CHROMAV_DC] =
716     pf->coeff_level_run[DCT_CHROMAU_4x4] = pf->coeff_level_run[DCT_CHROMAV_4x4] = pf->coeff_level_run[DCT_LUMA_4x4];
717     pf->coeff_level_run[DCT_CHROMA_AC]   = pf->coeff_level_run[DCT_CHROMAU_AC]  =
718     pf->coeff_level_run[DCT_CHROMAV_AC]  = pf->coeff_level_run[DCT_LUMA_AC];
719 }