Add partially accelerated mc_luma
[x262.git] / common / quant.c
1 /*****************************************************************************
2  * quant.c: quantization and level-run
3  *****************************************************************************
4  * Copyright (C) 2005-2014 x264 project
5  *
6  * Authors: Loren Merritt <lorenm@u.washington.edu>
7  *          Jason Garrett-Glaser <darkshikari@gmail.com>
8  *          Christian Heine <sennindemokrit@gmx.net>
9  *          Henrik Gramner <henrik@gramner.com>
10  *
11  * This program is free software; you can redistribute it and/or modify
12  * it under the terms of the GNU General Public License as published by
13  * the Free Software Foundation; either version 2 of the License, or
14  * (at your option) any later version.
15  *
16  * This program is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19  * GNU General Public License for more details.
20  *
21  * You should have received a copy of the GNU General Public License
22  * along with this program; if not, write to the Free Software
23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
24  *
25  * This program is also available under a commercial proprietary license.
26  * For more information, contact us at licensing@x264.com.
27  *****************************************************************************/
28
29 #include "common.h"
30
31 #if HAVE_MMX
32 #include "x86/quant.h"
33 #endif
34 #if ARCH_PPC
35 #   include "ppc/quant.h"
36 #endif
37 #if ARCH_ARM
38 #   include "arm/quant.h"
39 #endif
40
41 #define QUANT_ONE( coef, mf, f ) \
42 { \
43     if( (coef) > 0 ) \
44         (coef) = (f + (coef)) * (mf) >> 16; \
45     else \
46         (coef) = - ((f - (coef)) * (mf) >> 16); \
47     nz |= (coef); \
48 }
49
50 static int quant_8x8( dctcoef dct[64], udctcoef mf[64], udctcoef bias[64] )
51 {
52     int nz = 0;
53     for( int i = 0; i < 64; i++ )
54         QUANT_ONE( dct[i], mf[i], bias[i] );
55     return !!nz;
56 }
57
58 static int quant_4x4( dctcoef dct[16], udctcoef mf[16], udctcoef bias[16] )
59 {
60     int nz = 0;
61     for( int i = 0; i < 16; i++ )
62         QUANT_ONE( dct[i], mf[i], bias[i] );
63     return !!nz;
64 }
65
66 static int quant_4x4x4( dctcoef dct[4][16], udctcoef mf[16], udctcoef bias[16] )
67 {
68     int nza = 0;
69     for( int j = 0; j < 4; j++ )
70     {
71         int nz = 0;
72         for( int i = 0; i < 16; i++ )
73             QUANT_ONE( dct[j][i], mf[i], bias[i] );
74         nza |= (!!nz)<<j;
75     }
76     return nza;
77 }
78
79 static int quant_4x4_dc( dctcoef dct[16], int mf, int bias )
80 {
81     int nz = 0;
82     for( int i = 0; i < 16; i++ )
83         QUANT_ONE( dct[i], mf, bias );
84     return !!nz;
85 }
86
87 static int quant_2x2_dc( dctcoef dct[4], int mf, int bias )
88 {
89     int nz = 0;
90     QUANT_ONE( dct[0], mf, bias );
91     QUANT_ONE( dct[1], mf, bias );
92     QUANT_ONE( dct[2], mf, bias );
93     QUANT_ONE( dct[3], mf, bias );
94     return !!nz;
95 }
96
97 #define DEQUANT_SHL( x ) \
98     dct[x] = ( dct[x] * dequant_mf[i_mf][x] ) << i_qbits
99
100 #define DEQUANT_SHR( x ) \
101     dct[x] = ( dct[x] * dequant_mf[i_mf][x] + f ) >> (-i_qbits)
102
103 static void dequant_4x4( dctcoef dct[16], int dequant_mf[6][16], int i_qp )
104 {
105     const int i_mf = i_qp%6;
106     const int i_qbits = i_qp/6 - 4;
107
108     if( i_qbits >= 0 )
109     {
110         for( int i = 0; i < 16; i++ )
111             DEQUANT_SHL( i );
112     }
113     else
114     {
115         const int f = 1 << (-i_qbits-1);
116         for( int i = 0; i < 16; i++ )
117             DEQUANT_SHR( i );
118     }
119 }
120
121 static void dequant_8x8( dctcoef dct[64], int dequant_mf[6][64], int i_qp )
122 {
123     const int i_mf = i_qp%6;
124     const int i_qbits = i_qp/6 - 6;
125
126     if( i_qbits >= 0 )
127     {
128         for( int i = 0; i < 64; i++ )
129             DEQUANT_SHL( i );
130     }
131     else
132     {
133         const int f = 1 << (-i_qbits-1);
134         for( int i = 0; i < 64; i++ )
135             DEQUANT_SHR( i );
136     }
137 }
138
139 static void dequant_mpeg2_inter( dctcoef dct[64], int dequant_mf[64] )
140 {
141     int sign, sum = 0;
142     for( int i = 0; i < 64; i++ )
143     {
144         if( dct[i] )
145         {
146             sign = dct[i] >> 15;
147             dct[i] = (( (dct[i]<<1) + (sign|1) ) * dequant_mf[i] + (sign & 63)) >> 6;
148             dct[i] = x264_clip3( dct[i], -2048, 2047 );
149             sum ^= dct[i];
150         }
151     }
152     /* mismatch control */
153     dct[63] ^= 1&~sum;
154 }
155
156 static void dequant_mpeg2_intra( dctcoef dct[64], int dequant_mf[64], int precision )
157 {
158     int sign = 0;
159     int sum = dct[0] = dct[0] << ( 3 - precision ); // DC dequant
160     for( int i = 1; i < 64; i++ )
161     {
162         sign = dct[i] >> 15;
163         dct[i] = (dct[i] * dequant_mf[i] + (sign & 31)) >> 5;
164         dct[i] = x264_clip3( dct[i], -2048, 2047 );
165         sum ^= dct[i];
166     }
167     /* mismatch control */
168     dct[63] ^= 1&~sum;
169 }
170
171 static void dequant_4x4_dc( dctcoef dct[16], int dequant_mf[6][16], int i_qp )
172 {
173     const int i_qbits = i_qp/6 - 6;
174
175     if( i_qbits >= 0 )
176     {
177         const int i_dmf = dequant_mf[i_qp%6][0] << i_qbits;
178         for( int i = 0; i < 16; i++ )
179             dct[i] *= i_dmf;
180     }
181     else
182     {
183         const int i_dmf = dequant_mf[i_qp%6][0];
184         const int f = 1 << (-i_qbits-1);
185         for( int i = 0; i < 16; i++ )
186             dct[i] = ( dct[i] * i_dmf + f ) >> (-i_qbits);
187     }
188 }
189
190 #define IDCT_DEQUANT_2X4_START \
191     int a0 = dct[0] + dct[1]; \
192     int a1 = dct[2] + dct[3]; \
193     int a2 = dct[4] + dct[5]; \
194     int a3 = dct[6] + dct[7]; \
195     int a4 = dct[0] - dct[1]; \
196     int a5 = dct[2] - dct[3]; \
197     int a6 = dct[4] - dct[5]; \
198     int a7 = dct[6] - dct[7]; \
199     int b0 = a0 + a1; \
200     int b1 = a2 + a3; \
201     int b2 = a4 + a5; \
202     int b3 = a6 + a7; \
203     int b4 = a0 - a1; \
204     int b5 = a2 - a3; \
205     int b6 = a4 - a5; \
206     int b7 = a6 - a7;
207
208 static void idct_dequant_2x4_dc( dctcoef dct[8], dctcoef dct4x4[8][16], int dequant_mf[6][16], int i_qp )
209 {
210     IDCT_DEQUANT_2X4_START
211     int dmf = dequant_mf[i_qp%6][0] << i_qp/6;
212     dct4x4[0][0] = ((b0 + b1) * dmf + 32) >> 6;
213     dct4x4[1][0] = ((b2 + b3) * dmf + 32) >> 6;
214     dct4x4[2][0] = ((b0 - b1) * dmf + 32) >> 6;
215     dct4x4[3][0] = ((b2 - b3) * dmf + 32) >> 6;
216     dct4x4[4][0] = ((b4 - b5) * dmf + 32) >> 6;
217     dct4x4[5][0] = ((b6 - b7) * dmf + 32) >> 6;
218     dct4x4[6][0] = ((b4 + b5) * dmf + 32) >> 6;
219     dct4x4[7][0] = ((b6 + b7) * dmf + 32) >> 6;
220 }
221
222 static void idct_dequant_2x4_dconly( dctcoef dct[8], int dequant_mf[6][16], int i_qp )
223 {
224     IDCT_DEQUANT_2X4_START
225     int dmf = dequant_mf[i_qp%6][0] << i_qp/6;
226     dct[0] = ((b0 + b1) * dmf + 32) >> 6;
227     dct[1] = ((b2 + b3) * dmf + 32) >> 6;
228     dct[2] = ((b0 - b1) * dmf + 32) >> 6;
229     dct[3] = ((b2 - b3) * dmf + 32) >> 6;
230     dct[4] = ((b4 - b5) * dmf + 32) >> 6;
231     dct[5] = ((b6 - b7) * dmf + 32) >> 6;
232     dct[6] = ((b4 + b5) * dmf + 32) >> 6;
233     dct[7] = ((b6 + b7) * dmf + 32) >> 6;
234 }
235
236 static ALWAYS_INLINE void optimize_chroma_idct_dequant_2x4( dctcoef out[8], dctcoef dct[8], int dmf )
237 {
238     IDCT_DEQUANT_2X4_START
239     out[0] = ((b0 + b1) * dmf + 2080) >> 6; /* 2080 = 32 + (32<<6) */
240     out[1] = ((b2 + b3) * dmf + 2080) >> 6;
241     out[2] = ((b0 - b1) * dmf + 2080) >> 6;
242     out[3] = ((b2 - b3) * dmf + 2080) >> 6;
243     out[4] = ((b4 - b5) * dmf + 2080) >> 6;
244     out[5] = ((b6 - b7) * dmf + 2080) >> 6;
245     out[6] = ((b4 + b5) * dmf + 2080) >> 6;
246     out[7] = ((b6 + b7) * dmf + 2080) >> 6;
247 }
248 #undef IDCT_DEQUANT_2X4_START
249
250 static ALWAYS_INLINE void optimize_chroma_idct_dequant_2x2( dctcoef out[4], dctcoef dct[4], int dmf )
251 {
252     int d0 = dct[0] + dct[1];
253     int d1 = dct[2] + dct[3];
254     int d2 = dct[0] - dct[1];
255     int d3 = dct[2] - dct[3];
256     out[0] = ((d0 + d1) * dmf >> 5) + 32;
257     out[1] = ((d0 - d1) * dmf >> 5) + 32;
258     out[2] = ((d2 + d3) * dmf >> 5) + 32;
259     out[3] = ((d2 - d3) * dmf >> 5) + 32;
260 }
261
262 static ALWAYS_INLINE int optimize_chroma_round( dctcoef *ref, dctcoef *dct, int dequant_mf, int chroma422 )
263 {
264     dctcoef out[8];
265
266     if( chroma422 )
267         optimize_chroma_idct_dequant_2x4( out, dct, dequant_mf );
268     else
269         optimize_chroma_idct_dequant_2x2( out, dct, dequant_mf );
270
271     int sum = 0;
272     for( int i = 0; i < (chroma422?8:4); i++ )
273         sum |= ref[i] ^ out[i];
274     return sum >> 6;
275 }
276
277 static ALWAYS_INLINE int optimize_chroma_dc_internal( dctcoef *dct, int dequant_mf, int chroma422 )
278 {
279     /* dequant_mf = h->dequant4_mf[CQM_4IC + b_inter][i_qp%6][0] << i_qp/6, max 32*64 */
280     dctcoef dct_orig[8];
281     int coeff, nz;
282
283     if( chroma422 )
284         optimize_chroma_idct_dequant_2x4( dct_orig, dct, dequant_mf );
285     else
286         optimize_chroma_idct_dequant_2x2( dct_orig, dct, dequant_mf );
287
288     /* If the DC coefficients already round to zero, terminate early. */
289     int sum = 0;
290     for( int i = 0; i < (chroma422?8:4); i++ )
291         sum |= dct_orig[i];
292     if( !(sum >> 6) )
293         return 0;
294
295     /* Start with the highest frequency coefficient... is this the best option? */
296     for( nz = 0, coeff = (chroma422?7:3); coeff >= 0; coeff-- )
297     {
298         int level = dct[coeff];
299         int sign = level>>31 | 1; /* dct[coeff] < 0 ? -1 : 1 */
300
301         while( level )
302         {
303             dct[coeff] = level - sign;
304             if( optimize_chroma_round( dct_orig, dct, dequant_mf, chroma422 ) )
305             {
306                 nz = 1;
307                 dct[coeff] = level;
308                 break;
309             }
310             level -= sign;
311         }
312     }
313
314     return nz;
315 }
316
317 static int optimize_chroma_2x2_dc( dctcoef dct[4], int dequant_mf )
318 {
319     return optimize_chroma_dc_internal( dct, dequant_mf, 0 );
320 }
321
322 static int optimize_chroma_2x4_dc( dctcoef dct[8], int dequant_mf )
323 {
324     return optimize_chroma_dc_internal( dct, dequant_mf, 1 );
325 }
326
327 static void x264_denoise_dct( dctcoef *dct, uint32_t *sum, udctcoef *offset, int size )
328 {
329     for( int i = 0; i < size; i++ )
330     {
331         int level = dct[i];
332         int sign = level>>31;
333         level = (level+sign)^sign;
334         sum[i] += level;
335         level -= offset[i];
336         dct[i] = level<0 ? 0 : (level^sign)-sign;
337     }
338 }
339
340 /* (ref: JVT-B118)
341  * x264_mb_decimate_score: given dct coeffs it returns a score to see if we could empty this dct coeffs
342  * to 0 (low score means set it to null)
343  * Used in inter macroblock (luma and chroma)
344  *  luma: for a 8x8 block: if score < 4 -> null
345  *        for the complete mb: if score < 6 -> null
346  *  chroma: for the complete mb: if score < 7 -> null
347  */
348
349 const uint8_t x264_decimate_table4[16] =
350 {
351     3,2,2,1,1,1,0,0,0,0,0,0,0,0,0,0
352 };
353 const uint8_t x264_decimate_table8[64] =
354 {
355     3,3,3,3,2,2,2,2,2,2,2,2,1,1,1,1,
356     1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,
357     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
358     0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
359 };
360
361 static int ALWAYS_INLINE x264_decimate_score_internal( dctcoef *dct, int i_max )
362 {
363     const uint8_t *ds_table = (i_max == 64) ? x264_decimate_table8 : x264_decimate_table4;
364     int i_score = 0;
365     int idx = i_max - 1;
366
367     while( idx >= 0 && dct[idx] == 0 )
368         idx--;
369     while( idx >= 0 )
370     {
371         int i_run;
372
373         if( (unsigned)(dct[idx--] + 1) > 2 )
374             return 9;
375
376         i_run = 0;
377         while( idx >= 0 && dct[idx] == 0 )
378         {
379             idx--;
380             i_run++;
381         }
382         i_score += ds_table[i_run];
383     }
384
385     return i_score;
386 }
387
388 static int x264_decimate_score15( dctcoef *dct )
389 {
390     return x264_decimate_score_internal( dct+1, 15 );
391 }
392 static int x264_decimate_score16( dctcoef *dct )
393 {
394     return x264_decimate_score_internal( dct, 16 );
395 }
396 static int x264_decimate_score64( dctcoef *dct )
397 {
398     return x264_decimate_score_internal( dct, 64 );
399 }
400
401 #define last(num)\
402 static int x264_coeff_last##num( dctcoef *l )\
403 {\
404     int i_last = num-1;\
405     while( i_last >= 0 && l[i_last] == 0 )\
406         i_last--;\
407     return i_last;\
408 }
409
410 last(4)
411 last(8)
412 last(15)
413 last(16)
414 last(64)
415
416 #define level_run(num)\
417 static int x264_coeff_level_run##num( dctcoef *dct, x264_run_level_t *runlevel )\
418 {\
419     int i_last = runlevel->last = x264_coeff_last##num(dct);\
420     int i_total = 0;\
421     int mask = 0;\
422     do\
423     {\
424         int r = 0;\
425         runlevel->level[i_total] = dct[i_last];\
426         mask |= 1 << (i_last);\
427         while( --i_last >= 0 && dct[i_last] == 0 )\
428             r++;\
429         runlevel->run[i_total++] = r;\
430     } while( i_last >= 0 );\
431     runlevel->mask = mask;\
432     return i_total;\
433 }
434
435 level_run(4)
436 level_run(8)
437 level_run(15)
438 level_run(16)
439 level_run(64)
440
441 #if ARCH_X86_64
442 #define INIT_TRELLIS(cpu)\
443     pf->trellis_cabac_4x4 = x264_trellis_cabac_4x4_##cpu;\
444     pf->trellis_cabac_8x8 = x264_trellis_cabac_8x8_##cpu;\
445     pf->trellis_cabac_4x4_psy = x264_trellis_cabac_4x4_psy_##cpu;\
446     pf->trellis_cabac_8x8_psy = x264_trellis_cabac_8x8_psy_##cpu;\
447     pf->trellis_cabac_dc = x264_trellis_cabac_dc_##cpu;\
448     pf->trellis_cabac_chroma_422_dc = x264_trellis_cabac_chroma_422_dc_##cpu;
449 #else
450 #define INIT_TRELLIS(...)
451 #endif
452
453 void x264_quant_init( x264_t *h, int cpu, x264_quant_function_t *pf )
454 {
455     pf->quant_8x8 = quant_8x8;
456     pf->quant_4x4 = quant_4x4;
457     pf->quant_4x4x4 = quant_4x4x4;
458     pf->quant_4x4_dc = quant_4x4_dc;
459     pf->quant_2x2_dc = quant_2x2_dc;
460
461     pf->dequant_4x4 = dequant_4x4;
462     pf->dequant_4x4_dc = dequant_4x4_dc;
463     pf->dequant_8x8 = dequant_8x8;
464
465     pf->idct_dequant_2x4_dc = idct_dequant_2x4_dc;
466     pf->idct_dequant_2x4_dconly = idct_dequant_2x4_dconly;
467
468     pf->optimize_chroma_2x2_dc = optimize_chroma_2x2_dc;
469     pf->optimize_chroma_2x4_dc = optimize_chroma_2x4_dc;
470
471     pf->denoise_dct = x264_denoise_dct;
472     pf->decimate_score15 = x264_decimate_score15;
473     pf->decimate_score16 = x264_decimate_score16;
474     pf->decimate_score64 = x264_decimate_score64;
475
476     pf->coeff_last4 = x264_coeff_last4;
477     pf->coeff_last8 = x264_coeff_last8;
478     pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15;
479     pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16;
480     pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64;
481     pf->coeff_level_run4 = x264_coeff_level_run4;
482     pf->coeff_level_run8 = x264_coeff_level_run8;
483     pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15;
484     pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16;
485
486     pf->coeff_level_run[ DCT_LUMA_8x8] = x264_coeff_level_run64;
487     pf->dequant_mpeg2_intra = dequant_mpeg2_intra;
488     pf->dequant_mpeg2_inter = dequant_mpeg2_inter;
489
490 #if HIGH_BIT_DEPTH
491 #if HAVE_MMX
492     INIT_TRELLIS( sse2 );
493     if( cpu&X264_CPU_MMX2 )
494     {
495 #if ARCH_X86
496         pf->denoise_dct = x264_denoise_dct_mmx;
497         pf->decimate_score15 = x264_decimate_score15_mmx2;
498         pf->decimate_score16 = x264_decimate_score16_mmx2;
499         pf->decimate_score64 = x264_decimate_score64_mmx2;
500         pf->coeff_last8 = x264_coeff_last8_mmx2;
501         pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15_mmx2;
502         pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16_mmx2;
503         pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64_mmx2;
504         pf->coeff_level_run8 = x264_coeff_level_run8_mmx2;
505         pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15_mmx2;
506         pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16_mmx2;
507 #endif
508         pf->coeff_last4 = x264_coeff_last4_mmx2;
509         pf->coeff_level_run4 = x264_coeff_level_run4_mmx2;
510         if( cpu&X264_CPU_LZCNT )
511             pf->coeff_level_run4 = x264_coeff_level_run4_mmx2_lzcnt;
512     }
513     if( cpu&X264_CPU_SSE2 )
514     {
515         pf->quant_4x4 = x264_quant_4x4_sse2;
516         pf->quant_4x4x4 = x264_quant_4x4x4_sse2;
517         pf->quant_8x8 = x264_quant_8x8_sse2;
518         pf->quant_2x2_dc = x264_quant_2x2_dc_sse2;
519         pf->quant_4x4_dc = x264_quant_4x4_dc_sse2;
520         pf->dequant_4x4 = x264_dequant_4x4_sse2;
521         pf->dequant_8x8 = x264_dequant_8x8_sse2;
522         pf->dequant_4x4_dc = x264_dequant_4x4dc_sse2;
523         pf->denoise_dct = x264_denoise_dct_sse2;
524         pf->decimate_score15 = x264_decimate_score15_sse2;
525         pf->decimate_score16 = x264_decimate_score16_sse2;
526         pf->decimate_score64 = x264_decimate_score64_sse2;
527         pf->coeff_last8 = x264_coeff_last8_sse2;
528         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2;
529         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2;
530         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2;
531         pf->coeff_level_run8 = x264_coeff_level_run8_sse2;
532         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2;
533         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2;
534         if( cpu&X264_CPU_LZCNT )
535         {
536             pf->coeff_last4 = x264_coeff_last4_mmx2_lzcnt;
537             pf->coeff_last8 = x264_coeff_last8_sse2_lzcnt;
538             pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2_lzcnt;
539             pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2_lzcnt;
540             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2_lzcnt;
541             pf->coeff_level_run8 = x264_coeff_level_run8_sse2_lzcnt;
542             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2_lzcnt;
543             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2_lzcnt;
544         }
545     }
546     if( cpu&X264_CPU_SSSE3 )
547     {
548         pf->quant_4x4 = x264_quant_4x4_ssse3;
549         pf->quant_4x4x4 = x264_quant_4x4x4_ssse3;
550         pf->quant_8x8 = x264_quant_8x8_ssse3;
551         pf->quant_2x2_dc = x264_quant_2x2_dc_ssse3;
552         pf->quant_4x4_dc = x264_quant_4x4_dc_ssse3;
553         pf->denoise_dct = x264_denoise_dct_ssse3;
554         pf->decimate_score15 = x264_decimate_score15_ssse3;
555         pf->decimate_score16 = x264_decimate_score16_ssse3;
556         pf->decimate_score64 = x264_decimate_score64_ssse3;
557         INIT_TRELLIS( ssse3 );
558     }
559     if( cpu&X264_CPU_SSE4 )
560     {
561         pf->quant_2x2_dc = x264_quant_2x2_dc_sse4;
562         pf->quant_4x4_dc = x264_quant_4x4_dc_sse4;
563         pf->quant_4x4 = x264_quant_4x4_sse4;
564         pf->quant_4x4x4 = x264_quant_4x4x4_sse4;
565         pf->quant_8x8 = x264_quant_8x8_sse4;
566     }
567     if( cpu&X264_CPU_AVX )
568     {
569         pf->denoise_dct = x264_denoise_dct_avx;
570     }
571     if( cpu&X264_CPU_XOP )
572     {
573         pf->dequant_4x4_dc = x264_dequant_4x4dc_xop;
574         if( h->param.i_cqm_preset != X264_CQM_FLAT )
575         {
576             pf->dequant_4x4 = x264_dequant_4x4_xop;
577             pf->dequant_8x8 = x264_dequant_8x8_xop;
578         }
579     }
580     if( cpu&X264_CPU_AVX2 )
581     {
582         pf->quant_4x4 = x264_quant_4x4_avx2;
583         pf->quant_4x4_dc = x264_quant_4x4_dc_avx2;
584         pf->quant_8x8 = x264_quant_8x8_avx2;
585         pf->quant_4x4x4 = x264_quant_4x4x4_avx2;
586         pf->dequant_4x4 = x264_dequant_4x4_avx2;
587         pf->dequant_8x8 = x264_dequant_8x8_avx2;
588         pf->dequant_4x4_dc = x264_dequant_4x4dc_avx2;
589         pf->denoise_dct = x264_denoise_dct_avx2;
590     }
591 #endif // HAVE_MMX
592 #else // !HIGH_BIT_DEPTH
593 #if HAVE_MMX
594     INIT_TRELLIS( sse2 );
595     if( cpu&X264_CPU_MMX )
596     {
597 #if ARCH_X86
598         pf->quant_4x4 = x264_quant_4x4_mmx;
599         pf->quant_4x4x4 = x264_quant_4x4x4_mmx;
600         pf->quant_8x8 = x264_quant_8x8_mmx;
601         pf->dequant_4x4 = x264_dequant_4x4_mmx;
602         pf->dequant_4x4_dc = x264_dequant_4x4dc_mmx2;
603         pf->dequant_8x8 = x264_dequant_8x8_mmx;
604         if( h->param.i_cqm_preset == X264_CQM_FLAT )
605         {
606             pf->dequant_4x4 = x264_dequant_4x4_flat16_mmx;
607             pf->dequant_8x8 = x264_dequant_8x8_flat16_mmx;
608         }
609         pf->denoise_dct = x264_denoise_dct_mmx;
610 #endif
611     }
612
613     if( cpu&X264_CPU_MMX2 )
614     {
615         pf->quant_2x2_dc = x264_quant_2x2_dc_mmx2;
616 #if ARCH_X86
617         pf->quant_4x4_dc = x264_quant_4x4_dc_mmx2;
618         pf->decimate_score15 = x264_decimate_score15_mmx2;
619         pf->decimate_score16 = x264_decimate_score16_mmx2;
620         pf->decimate_score64 = x264_decimate_score64_mmx2;
621         pf->coeff_last[  DCT_LUMA_AC] = x264_coeff_last15_mmx2;
622         pf->coeff_last[ DCT_LUMA_4x4] = x264_coeff_last16_mmx2;
623         pf->coeff_last[ DCT_LUMA_8x8] = x264_coeff_last64_mmx2;
624         pf->coeff_level_run[  DCT_LUMA_AC] = x264_coeff_level_run15_mmx2;
625         pf->coeff_level_run[ DCT_LUMA_4x4] = x264_coeff_level_run16_mmx2;
626 #endif
627         pf->coeff_last4 = x264_coeff_last4_mmx2;
628         pf->coeff_last8 = x264_coeff_last8_mmx2;
629         pf->coeff_level_run4 = x264_coeff_level_run4_mmx2;
630         pf->coeff_level_run8 = x264_coeff_level_run8_mmx2;
631         if( cpu&X264_CPU_LZCNT )
632         {
633             pf->coeff_last4 = x264_coeff_last4_mmx2_lzcnt;
634             pf->coeff_last8 = x264_coeff_last8_mmx2_lzcnt;
635             pf->coeff_level_run4 = x264_coeff_level_run4_mmx2_lzcnt;
636             pf->coeff_level_run8 = x264_coeff_level_run8_mmx2_lzcnt;
637         }
638     }
639
640     if( cpu&X264_CPU_SSE2 )
641     {
642         pf->quant_4x4_dc = x264_quant_4x4_dc_sse2;
643         pf->quant_4x4 = x264_quant_4x4_sse2;
644         pf->quant_4x4x4 = x264_quant_4x4x4_sse2;
645         pf->quant_8x8 = x264_quant_8x8_sse2;
646         pf->dequant_4x4 = x264_dequant_4x4_sse2;
647         pf->dequant_4x4_dc = x264_dequant_4x4dc_sse2;
648         pf->dequant_8x8 = x264_dequant_8x8_sse2;
649         if( h->param.i_cqm_preset == X264_CQM_FLAT )
650         {
651             pf->dequant_4x4 = x264_dequant_4x4_flat16_sse2;
652             pf->dequant_8x8 = x264_dequant_8x8_flat16_sse2;
653         }
654         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_sse2;
655         pf->denoise_dct = x264_denoise_dct_sse2;
656         pf->decimate_score15 = x264_decimate_score15_sse2;
657         pf->decimate_score16 = x264_decimate_score16_sse2;
658         pf->decimate_score64 = x264_decimate_score64_sse2;
659         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2;
660         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2;
661         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2;
662         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2;
663         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2;
664         if( cpu&X264_CPU_LZCNT )
665         {
666             pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_sse2_lzcnt;
667             pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_sse2_lzcnt;
668             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_sse2_lzcnt;
669             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_sse2_lzcnt;
670             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_sse2_lzcnt;
671         }
672     }
673
674     if( cpu&X264_CPU_SSSE3 )
675     {
676         pf->quant_2x2_dc = x264_quant_2x2_dc_ssse3;
677         pf->quant_4x4_dc = x264_quant_4x4_dc_ssse3;
678         pf->quant_4x4 = x264_quant_4x4_ssse3;
679         pf->quant_4x4x4 = x264_quant_4x4x4_ssse3;
680         pf->quant_8x8 = x264_quant_8x8_ssse3;
681         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_ssse3;
682         pf->denoise_dct = x264_denoise_dct_ssse3;
683         pf->decimate_score15 = x264_decimate_score15_ssse3;
684         pf->decimate_score16 = x264_decimate_score16_ssse3;
685         pf->decimate_score64 = x264_decimate_score64_ssse3;
686         INIT_TRELLIS( ssse3 );
687         pf->coeff_level_run4 = x264_coeff_level_run4_ssse3;
688         pf->coeff_level_run8 = x264_coeff_level_run8_ssse3;
689         pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_ssse3;
690         pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_ssse3;
691         if( cpu&X264_CPU_LZCNT )
692         {
693             pf->coeff_level_run4 = x264_coeff_level_run4_ssse3;
694             pf->coeff_level_run8 = x264_coeff_level_run8_ssse3;
695             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_ssse3_lzcnt;
696             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_ssse3_lzcnt;
697         }
698     }
699
700     if( cpu&X264_CPU_SSE4 )
701     {
702         pf->quant_4x4_dc = x264_quant_4x4_dc_sse4;
703         pf->quant_4x4 = x264_quant_4x4_sse4;
704         pf->quant_8x8 = x264_quant_8x8_sse4;
705         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_sse4;
706     }
707
708     if( cpu&X264_CPU_AVX )
709     {
710         pf->dequant_4x4_dc = x264_dequant_4x4dc_avx;
711         if( h->param.i_cqm_preset != X264_CQM_FLAT )
712         {
713             pf->dequant_4x4 = x264_dequant_4x4_avx;
714             pf->dequant_8x8 = x264_dequant_8x8_avx;
715         }
716         pf->optimize_chroma_2x2_dc = x264_optimize_chroma_2x2_dc_avx;
717         pf->denoise_dct = x264_denoise_dct_avx;
718     }
719
720     if( cpu&X264_CPU_XOP )
721     {
722         if( h->param.i_cqm_preset != X264_CQM_FLAT )
723         {
724             pf->dequant_4x4 = x264_dequant_4x4_xop;
725             pf->dequant_8x8 = x264_dequant_8x8_xop;
726         }
727     }
728
729     if( cpu&X264_CPU_AVX2 )
730     {
731         pf->quant_4x4 = x264_quant_4x4_avx2;
732         pf->quant_4x4_dc = x264_quant_4x4_dc_avx2;
733         pf->quant_8x8 = x264_quant_8x8_avx2;
734         pf->quant_4x4x4 = x264_quant_4x4x4_avx2;
735         pf->dequant_4x4 = x264_dequant_4x4_avx2;
736         pf->dequant_8x8 = x264_dequant_8x8_avx2;
737         pf->dequant_4x4_dc = x264_dequant_4x4dc_avx2;
738         if( h->param.i_cqm_preset == X264_CQM_FLAT )
739         {
740             pf->dequant_4x4 = x264_dequant_4x4_flat16_avx2;
741             pf->dequant_8x8 = x264_dequant_8x8_flat16_avx2;
742         }
743         pf->decimate_score64 = x264_decimate_score64_avx2;
744         pf->denoise_dct = x264_denoise_dct_avx2;
745         if( cpu&X264_CPU_LZCNT )
746         {
747             pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_avx2_lzcnt;
748             pf->coeff_level_run[ DCT_LUMA_AC] = x264_coeff_level_run15_avx2_lzcnt;
749             pf->coeff_level_run[DCT_LUMA_4x4] = x264_coeff_level_run16_avx2_lzcnt;
750         }
751     }
752 #endif // HAVE_MMX
753
754 #if HAVE_ALTIVEC
755     if( cpu&X264_CPU_ALTIVEC ) {
756         pf->quant_2x2_dc = x264_quant_2x2_dc_altivec;
757         pf->quant_4x4_dc = x264_quant_4x4_dc_altivec;
758         pf->quant_4x4 = x264_quant_4x4_altivec;
759         pf->quant_8x8 = x264_quant_8x8_altivec;
760
761         pf->dequant_4x4 = x264_dequant_4x4_altivec;
762         pf->dequant_8x8 = x264_dequant_8x8_altivec;
763     }
764 #endif
765
766 #if HAVE_ARMV6
767     if( cpu&X264_CPU_ARMV6 )
768         pf->coeff_last4 = x264_coeff_last4_arm;
769
770     if( cpu&X264_CPU_NEON )
771     {
772         pf->quant_2x2_dc   = x264_quant_2x2_dc_neon;
773         pf->quant_4x4      = x264_quant_4x4_neon;
774         pf->quant_4x4_dc   = x264_quant_4x4_dc_neon;
775         pf->quant_4x4x4    = x264_quant_4x4x4_neon;
776         pf->quant_8x8      = x264_quant_8x8_neon;
777         pf->dequant_4x4    = x264_dequant_4x4_neon;
778         pf->dequant_4x4_dc = x264_dequant_4x4_dc_neon;
779         pf->dequant_8x8    = x264_dequant_8x8_neon;
780         pf->coeff_last[ DCT_LUMA_AC] = x264_coeff_last15_neon;
781         pf->coeff_last[DCT_LUMA_4x4] = x264_coeff_last16_neon;
782         pf->coeff_last[DCT_LUMA_8x8] = x264_coeff_last64_neon;
783         pf->dequant_mpeg2_inter = x264_dequant_mpeg2_inter_neon;
784         pf->dequant_mpeg2_intra = x264_dequant_mpeg2_intra_neon;
785     }
786 #endif
787 #endif // HIGH_BIT_DEPTH
788     pf->coeff_last[DCT_LUMA_DC]     = pf->coeff_last[DCT_CHROMAU_DC]  = pf->coeff_last[DCT_CHROMAV_DC] =
789     pf->coeff_last[DCT_CHROMAU_4x4] = pf->coeff_last[DCT_CHROMAV_4x4] = pf->coeff_last[DCT_LUMA_4x4];
790     pf->coeff_last[DCT_CHROMA_AC]   = pf->coeff_last[DCT_CHROMAU_AC]  =
791     pf->coeff_last[DCT_CHROMAV_AC]  = pf->coeff_last[DCT_LUMA_AC];
792     pf->coeff_last[DCT_CHROMAU_8x8] = pf->coeff_last[DCT_CHROMAV_8x8] = pf->coeff_last[DCT_LUMA_8x8];
793
794     pf->coeff_level_run[DCT_LUMA_DC]     = pf->coeff_level_run[DCT_CHROMAU_DC]  = pf->coeff_level_run[DCT_CHROMAV_DC] =
795     pf->coeff_level_run[DCT_CHROMAU_4x4] = pf->coeff_level_run[DCT_CHROMAV_4x4] = pf->coeff_level_run[DCT_LUMA_4x4];
796     pf->coeff_level_run[DCT_CHROMA_AC]   = pf->coeff_level_run[DCT_CHROMAU_AC]  =
797     pf->coeff_level_run[DCT_CHROMAV_AC]  = pf->coeff_level_run[DCT_LUMA_AC];
798 }