x86: AVX2 predict_16x16_p
[x264/x264-sandbox.git] / common / x86 / predict-c.c
1 /*****************************************************************************
2  * predict-c.c: intra prediction
3  *****************************************************************************
4  * Copyright (C) 2003-2013 x264 project
5  *
6  * Authors: Laurent Aimar <fenrir@via.ecp.fr>
7  *          Loren Merritt <lorenm@u.washington.edu>
8  *          Fiona Glaser <fiona@x264.com>
9  *
10  * This program is free software; you can redistribute it and/or modify
11  * it under the terms of the GNU General Public License as published by
12  * the Free Software Foundation; either version 2 of the License, or
13  * (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  * GNU General Public License for more details.
19  *
20  * You should have received a copy of the GNU General Public License
21  * along with this program; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
23  *
24  * This program is also available under a commercial proprietary license.
25  * For more information, contact us at licensing@x264.com.
26  *****************************************************************************/
27
28 #include "common/common.h"
29 #include "predict.h"
30 #include "pixel.h"
31
32 #define PREDICT_16x16_DC(name)\
33 void x264_predict_16x16_dc_##name( pixel *src )\
34 {\
35     uint32_t dc = 16;\
36     for( int i = 0; i < 16; i += 2 )\
37     {\
38         dc += src[-1 + i * FDEC_STRIDE];\
39         dc += src[-1 + (i+1) * FDEC_STRIDE];\
40     }\
41     x264_predict_16x16_dc_core_##name( src, dc );\
42 }
43
44 PREDICT_16x16_DC( mmx2 )
45 PREDICT_16x16_DC( sse2 )
46
47 #define PREDICT_16x16_DC_LEFT(name)\
48 static void x264_predict_16x16_dc_left_##name( pixel *src )\
49 {\
50     uint32_t dc = 8;\
51     for( int i = 0; i < 16; i += 2 )\
52     {\
53         dc += src[-1 + i * FDEC_STRIDE];\
54         dc += src[-1 + (i+1) * FDEC_STRIDE];\
55     }\
56     x264_predict_16x16_dc_left_core_##name( src, dc>>4 );\
57 }
58
59 PREDICT_16x16_DC_LEFT( mmx2 )
60 PREDICT_16x16_DC_LEFT( sse2 )
61
62 #define PREDICT_P_SUM(j,i)\
63     H += i * ( src[j+i - FDEC_STRIDE ]  - src[j-i - FDEC_STRIDE ] );\
64     V += i * ( src[(j+i)*FDEC_STRIDE -1] - src[(j-i)*FDEC_STRIDE -1] );\
65
66 ALIGNED_16( static const int16_t pw_12345678[8] ) = {1,2,3,4,5,6,7,8};
67 ALIGNED_16( static const int16_t pw_m87654321[8] ) = {-8,-7,-6,-5,-4,-3,-2,-1};
68 ALIGNED_16( static const int16_t pw_m32101234[8] ) = {-3,-2,-1,0,1,2,3,4};
69 ALIGNED_8( static const int8_t pb_12345678[8] ) = {1,2,3,4,5,6,7,8};
70 ALIGNED_8( static const int8_t pb_m87654321[8] ) = {-8,-7,-6,-5,-4,-3,-2,-1};
71 ALIGNED_8( static const int8_t pb_m32101234[8] ) = {-3,-2,-1,0,1,2,3,4};
72
73 #if !HIGH_BIT_DEPTH
74 #define PREDICT_16x16_P(name)\
75 static void x264_predict_16x16_p_##name( pixel *src )\
76 {\
77     int a, b, c;\
78     int H = 0;\
79     int V = 0;\
80     int i00;\
81     PREDICT_P_SUM(7,1) \
82     PREDICT_P_SUM(7,2) \
83     PREDICT_P_SUM(7,3) \
84     PREDICT_P_SUM(7,4) \
85     PREDICT_P_SUM(7,5) \
86     PREDICT_P_SUM(7,6) \
87     PREDICT_P_SUM(7,7) \
88     PREDICT_P_SUM(7,8) \
89     a = 16 * ( src[15*FDEC_STRIDE -1] + src[15 - FDEC_STRIDE] );\
90     b = ( 5 * H + 32 ) >> 6;\
91     c = ( 5 * V + 32 ) >> 6;\
92     i00 = a - b * 7 - c * 7 + 16;\
93     x264_predict_16x16_p_core_##name( src, i00, b, c );\
94 }
95 #ifndef ARCH_X86_64
96 PREDICT_16x16_P(mmx2)
97 #endif
98 PREDICT_16x16_P(sse2)
99 #endif //!HIGH_BIT_DEPTH
100
101 #define PREDICT_8x16C_P_CORE \
102     int H = 0, V = 0;\
103     for( int i = 0; i < 4; i++ )\
104         H += ( i + 1 ) * ( src[4 + i - FDEC_STRIDE] - src[2 - i - FDEC_STRIDE] );\
105     for( int i = 0; i < 8; i++ )\
106         V += ( i + 1 ) * ( src[-1 + (i+8)*FDEC_STRIDE] - src[-1 + (6-i)*FDEC_STRIDE] );\
107     int a = 16 * ( src[-1 + 15*FDEC_STRIDE] + src[7 - FDEC_STRIDE] );\
108     int b = ( 17 * H + 16 ) >> 5;\
109     int c = ( 5 * V + 32 ) >> 6;
110
111 #if HIGH_BIT_DEPTH
112 #define PREDICT_8x16_P(name)\
113 static void x264_predict_8x16c_p_##name( uint16_t *src )\
114 {\
115     PREDICT_8x16C_P_CORE \
116     x264_predict_8x16c_p_core_##name( src, a, b, c );\
117 }
118
119 PREDICT_8x16_P(sse2)
120 PREDICT_8x16_P(avx)
121 #else
122 #define PREDICT_8x16_P(name)\
123 static void x264_predict_8x16c_p_##name( uint8_t *src )\
124 {\
125     PREDICT_8x16C_P_CORE \
126     int i00 = a -3*b -7*c + 16;\
127     x264_predict_8x16c_p_core_##name( src, i00, b, c );\
128 }
129 #ifndef ARCH_X86_64
130 PREDICT_8x16_P(mmx2)
131 #endif
132 PREDICT_8x16_P(sse2)
133 PREDICT_8x16_P(avx)
134 #endif
135
136 #if HAVE_X86_INLINE_ASM
137 #if HIGH_BIT_DEPTH
138 #define PREDICT_16x16_P_ASM\
139     asm (\
140         "movdqu           %1, %%xmm1 \n"\
141         "movdqa           %2, %%xmm0 \n"\
142         "pmaddwd          %3, %%xmm0 \n"\
143         "pmaddwd          %4, %%xmm1 \n"\
144         "paddd        %%xmm1, %%xmm0 \n"\
145         "movhlps      %%xmm0, %%xmm1 \n"\
146         "paddd        %%xmm1, %%xmm0 \n"\
147         "pshuflw $14, %%xmm0, %%xmm1 \n"\
148         "paddd        %%xmm1, %%xmm0 \n"\
149         "movd         %%xmm0, %0     \n"\
150         :"=r"(H)\
151         :"m"(src[-FDEC_STRIDE-1]), "m"(src[-FDEC_STRIDE+8]),\
152          "m"(*pw_12345678), "m"(*pw_m87654321)\
153     );
154 #else
155 #define PREDICT_16x16_P_ASM\
156     asm (\
157         "movq           %1, %%mm1 \n"\
158         "movq           %2, %%mm0 \n"\
159         "palignr $7,    %3, %%mm1 \n"\
160         "pmaddubsw      %4, %%mm0 \n"\
161         "pmaddubsw      %5, %%mm1 \n"\
162         "paddw       %%mm1, %%mm0 \n"\
163         "pshufw $14, %%mm0, %%mm1 \n"\
164         "paddw       %%mm1, %%mm0 \n"\
165         "pshufw  $1, %%mm0, %%mm1 \n"\
166         "paddw       %%mm1, %%mm0 \n"\
167         "movd        %%mm0, %0    \n"\
168         "movswl        %w0, %0    \n"\
169         :"=r"(H)\
170         :"m"(src[-FDEC_STRIDE]), "m"(src[-FDEC_STRIDE+8]),\
171          "m"(src[-FDEC_STRIDE-8]), "m"(*pb_12345678), "m"(*pb_m87654321)\
172     );
173 #endif
174 #define PREDICT_16x16_P_INLINE(name, name2)\
175 static void x264_predict_16x16_p_##name( pixel *src )\
176 {\
177     int a, b, c, i00;\
178     int H, V;\
179     PREDICT_16x16_P_ASM\
180     V = 8 * ( src[15*FDEC_STRIDE-1] - src[-1*FDEC_STRIDE-1] )\
181       + 7 * ( src[14*FDEC_STRIDE-1] - src[ 0*FDEC_STRIDE-1] )\
182       + 6 * ( src[13*FDEC_STRIDE-1] - src[ 1*FDEC_STRIDE-1] )\
183       + 5 * ( src[12*FDEC_STRIDE-1] - src[ 2*FDEC_STRIDE-1] )\
184       + 4 * ( src[11*FDEC_STRIDE-1] - src[ 3*FDEC_STRIDE-1] )\
185       + 3 * ( src[10*FDEC_STRIDE-1] - src[ 4*FDEC_STRIDE-1] )\
186       + 2 * ( src[ 9*FDEC_STRIDE-1] - src[ 5*FDEC_STRIDE-1] )\
187       + 1 * ( src[ 8*FDEC_STRIDE-1] - src[ 6*FDEC_STRIDE-1] );\
188     a = 16 * ( src[15*FDEC_STRIDE -1] + src[15 - FDEC_STRIDE] );\
189     b = ( 5 * H + 32 ) >> 6;\
190     c = ( 5 * V + 32 ) >> 6;\
191     i00 = a - b * 7 - c * 7 + 16;\
192     /* b*15 + c*15 can overflow: it's easier to just branch away in this rare case
193      * than to try to consider it in the asm. */\
194     if( BIT_DEPTH > 8 && (i00 > 0x7fff || abs(b) > 1092 || abs(c) > 1092) )\
195         x264_predict_16x16_p_c( src );\
196     else\
197         x264_predict_16x16_p_core_##name2( src, i00, b, c );\
198 }
199 #if HIGH_BIT_DEPTH
200 PREDICT_16x16_P_INLINE( sse2, sse2 )
201 #else
202 PREDICT_16x16_P_INLINE( ssse3, sse2 )
203 PREDICT_16x16_P_INLINE( avx, avx )
204 #endif
205 PREDICT_16x16_P_INLINE( avx2, avx2 )
206 #endif
207
208 #if !HIGH_BIT_DEPTH
209
210 #define PREDICT_8x8_P(name)\
211 static void x264_predict_8x8c_p_##name( uint8_t *src )\
212 {\
213     int a, b, c;\
214     int H = 0;\
215     int V = 0;\
216     int i00;\
217     PREDICT_P_SUM(3,1)\
218     PREDICT_P_SUM(3,2)\
219     PREDICT_P_SUM(3,3)\
220     PREDICT_P_SUM(3,4)\
221     a = 16 * ( src[7*FDEC_STRIDE -1] + src[7 - FDEC_STRIDE] );\
222     b = ( 17 * H + 16 ) >> 5;\
223     c = ( 17 * V + 16 ) >> 5;\
224     i00 = a -3*b -3*c + 16;\
225     x264_predict_8x8c_p_core_##name( src, i00, b, c );\
226 }
227 #ifndef ARCH_X86_64
228 PREDICT_8x8_P( mmx2 )
229 #endif
230 PREDICT_8x8_P( sse2   )
231
232 #endif //!HIGH_BIT_DEPTH
233
234 #if HAVE_X86_INLINE_ASM
235
236 #define PREDICT_8x8C_P_CORE\
237     V = 1 * ( src[4*FDEC_STRIDE -1] - src[ 2*FDEC_STRIDE -1] )\
238       + 2 * ( src[5*FDEC_STRIDE -1] - src[ 1*FDEC_STRIDE -1] )\
239       + 3 * ( src[6*FDEC_STRIDE -1] - src[ 0*FDEC_STRIDE -1] )\
240       + 4 * ( src[7*FDEC_STRIDE -1] - src[-1*FDEC_STRIDE -1] );\
241     H += -4 * src[-1*FDEC_STRIDE -1];\
242     int a = 16 * ( src[7*FDEC_STRIDE -1] + src[7 - FDEC_STRIDE] );\
243     int b = ( 17 * H + 16 ) >> 5;\
244     int c = ( 17 * V + 16 ) >> 5;
245
246 #if HIGH_BIT_DEPTH
247 #define PREDICT_8x8_P2(cpu1, cpu2)\
248 static void x264_predict_8x8c_p_ ## cpu1( pixel *src )\
249 {\
250     int H, V;\
251     asm (\
252         "movdqa           %1, %%xmm0 \n"\
253         "pmaddwd          %2, %%xmm0 \n"\
254         "movhlps      %%xmm0, %%xmm1 \n"\
255         "paddd        %%xmm1, %%xmm0 \n"\
256         "pshuflw $14, %%xmm0, %%xmm1 \n"\
257         "paddd        %%xmm1, %%xmm0 \n"\
258         "movd         %%xmm0, %0     \n"\
259         :"=r"(H)\
260         :"m"(src[-FDEC_STRIDE]), "m"(*pw_m32101234)\
261     );\
262     PREDICT_8x8C_P_CORE\
263     x264_predict_8x8c_p_core_ ## cpu2( src, a, b, c );\
264 }
265
266 PREDICT_8x8_P2(sse2, sse2)
267 PREDICT_8x8_P2( avx,  avx)
268
269 #else  //!HIGH_BIT_DEPTH
270 #define PREDICT_8x8_P2(cpu1, cpu2)\
271 static void x264_predict_8x8c_p_ ## cpu1( pixel *src )\
272 {\
273     int H, V;\
274     asm (\
275         "movq           %1, %%mm0 \n"\
276         "pmaddubsw      %2, %%mm0 \n"\
277         "pshufw $14, %%mm0, %%mm1 \n"\
278         "paddw       %%mm1, %%mm0 \n"\
279         "pshufw  $1, %%mm0, %%mm1 \n"\
280         "paddw       %%mm1, %%mm0 \n"\
281         "movd        %%mm0, %0    \n"\
282         "movswl        %w0, %0    \n"\
283         :"=r"(H)\
284         :"m"(src[-FDEC_STRIDE]), "m"(*pb_m32101234)\
285     );\
286     PREDICT_8x8C_P_CORE\
287     int i00 = a -3*b -3*c + 16;\
288     x264_predict_8x8c_p_core_ ## cpu2( src, i00, b, c );\
289 }
290
291 PREDICT_8x8_P2(ssse3, sse2)
292 PREDICT_8x8_P2(  avx,  avx)
293 #endif
294 #endif
295
296 #if ARCH_X86_64 && !HIGH_BIT_DEPTH
297 static void x264_predict_8x8c_dc_left( uint8_t *src )
298 {
299     int y;
300     uint32_t s0 = 0, s1 = 0;
301     uint64_t dc0, dc1;
302
303     for( y = 0; y < 4; y++ )
304     {
305         s0 += src[y * FDEC_STRIDE     - 1];
306         s1 += src[(y+4) * FDEC_STRIDE - 1];
307     }
308     dc0 = (( s0 + 2 ) >> 2) * 0x0101010101010101ULL;
309     dc1 = (( s1 + 2 ) >> 2) * 0x0101010101010101ULL;
310
311     for( y = 0; y < 4; y++ )
312     {
313         M64( src ) = dc0;
314         src += FDEC_STRIDE;
315     }
316     for( y = 0; y < 4; y++ )
317     {
318         M64( src ) = dc1;
319         src += FDEC_STRIDE;
320     }
321
322 }
323 #endif // ARCH_X86_64 && !HIGH_BIT_DEPTH
324
325 /****************************************************************************
326  * Exported functions:
327  ****************************************************************************/
328 void x264_predict_16x16_init_mmx( int cpu, x264_predict_t pf[7] )
329 {
330     if( !(cpu&X264_CPU_MMX2) )
331         return;
332     pf[I_PRED_16x16_DC]      = x264_predict_16x16_dc_mmx2;
333     pf[I_PRED_16x16_DC_TOP]  = x264_predict_16x16_dc_top_mmx2;
334     pf[I_PRED_16x16_DC_LEFT] = x264_predict_16x16_dc_left_mmx2;
335     pf[I_PRED_16x16_V]       = x264_predict_16x16_v_mmx2;
336     pf[I_PRED_16x16_H]       = x264_predict_16x16_h_mmx2;
337 #if HIGH_BIT_DEPTH
338     if( !(cpu&X264_CPU_SSE) )
339         return;
340     pf[I_PRED_16x16_V]       = x264_predict_16x16_v_sse;
341     if( !(cpu&X264_CPU_SSE2) )
342         return;
343     pf[I_PRED_16x16_DC]      = x264_predict_16x16_dc_sse2;
344     pf[I_PRED_16x16_DC_TOP]  = x264_predict_16x16_dc_top_sse2;
345     pf[I_PRED_16x16_DC_LEFT] = x264_predict_16x16_dc_left_sse2;
346     pf[I_PRED_16x16_H]       = x264_predict_16x16_h_sse2;
347 #if HAVE_X86_INLINE_ASM
348     pf[I_PRED_16x16_P]       = x264_predict_16x16_p_sse2;
349 #endif
350     if( !(cpu&X264_CPU_AVX) )
351         return;
352     pf[I_PRED_16x16_V]       = x264_predict_16x16_v_avx;
353     if( !(cpu&X264_CPU_AVX2) )
354         return;
355     pf[I_PRED_16x16_H]       = x264_predict_16x16_h_avx2;
356 #else
357 #if !ARCH_X86_64
358     pf[I_PRED_16x16_P]       = x264_predict_16x16_p_mmx2;
359 #endif
360     if( !(cpu&X264_CPU_SSE) )
361         return;
362     pf[I_PRED_16x16_V]       = x264_predict_16x16_v_sse;
363     if( !(cpu&X264_CPU_SSE2) )
364         return;
365     pf[I_PRED_16x16_DC]      = x264_predict_16x16_dc_sse2;
366     if( cpu&X264_CPU_SSE2_IS_SLOW )
367         return;
368     pf[I_PRED_16x16_DC_TOP]  = x264_predict_16x16_dc_top_sse2;
369     pf[I_PRED_16x16_DC_LEFT] = x264_predict_16x16_dc_left_sse2;
370     pf[I_PRED_16x16_P]       = x264_predict_16x16_p_sse2;
371     if( !(cpu&X264_CPU_SSSE3) )
372         return;
373     if( !(cpu&X264_CPU_SLOW_PSHUFB) )
374         pf[I_PRED_16x16_H]       = x264_predict_16x16_h_ssse3;
375 #if HAVE_X86_INLINE_ASM
376     pf[I_PRED_16x16_P]       = x264_predict_16x16_p_ssse3;
377 #endif
378     if( !(cpu&X264_CPU_AVX) )
379         return;
380     pf[I_PRED_16x16_P]       = x264_predict_16x16_p_avx;
381 #endif // HIGH_BIT_DEPTH
382
383     if( cpu&X264_CPU_AVX2 )
384     {
385         pf[I_PRED_16x16_P]       = x264_predict_16x16_p_avx2;
386     }
387 }
388
389 void x264_predict_8x8c_init_mmx( int cpu, x264_predict_t pf[7] )
390 {
391     if( !(cpu&X264_CPU_MMX) )
392         return;
393 #if HIGH_BIT_DEPTH
394     pf[I_PRED_CHROMA_V]       = x264_predict_8x8c_v_mmx;
395     if( !(cpu&X264_CPU_MMX2) )
396         return;
397     pf[I_PRED_CHROMA_DC]      = x264_predict_8x8c_dc_mmx2;
398     pf[I_PRED_CHROMA_H]       = x264_predict_8x8c_h_mmx2;
399     if( !(cpu&X264_CPU_SSE) )
400         return;
401     pf[I_PRED_CHROMA_V]       = x264_predict_8x8c_v_sse;
402     if( !(cpu&X264_CPU_SSE2) )
403         return;
404     pf[I_PRED_CHROMA_DC]      = x264_predict_8x8c_dc_sse2;
405     pf[I_PRED_CHROMA_DC_TOP]  = x264_predict_8x8c_dc_top_sse2;
406     pf[I_PRED_CHROMA_H]       = x264_predict_8x8c_h_sse2;
407 #if HAVE_X86_INLINE_ASM
408     pf[I_PRED_CHROMA_P]       = x264_predict_8x8c_p_sse2;
409     if( !(cpu&X264_CPU_AVX) )
410         return;
411     pf[I_PRED_CHROMA_P]       = x264_predict_8x8c_p_avx;
412     if( !(cpu&X264_CPU_AVX2) )
413         return;
414     pf[I_PRED_CHROMA_H]   = x264_predict_8x8c_h_avx2;
415 #endif
416 #else
417 #if ARCH_X86_64
418     pf[I_PRED_CHROMA_DC_LEFT] = x264_predict_8x8c_dc_left;
419 #endif
420     pf[I_PRED_CHROMA_V]       = x264_predict_8x8c_v_mmx;
421     if( !(cpu&X264_CPU_MMX2) )
422         return;
423     pf[I_PRED_CHROMA_DC_TOP]  = x264_predict_8x8c_dc_top_mmx2;
424     pf[I_PRED_CHROMA_H]       = x264_predict_8x8c_h_mmx2;
425 #if !ARCH_X86_64
426     pf[I_PRED_CHROMA_P]       = x264_predict_8x8c_p_mmx2;
427 #endif
428     pf[I_PRED_CHROMA_DC]      = x264_predict_8x8c_dc_mmx2;
429     if( !(cpu&X264_CPU_SSE2) )
430         return;
431     pf[I_PRED_CHROMA_P]       = x264_predict_8x8c_p_sse2;
432     if( !(cpu&X264_CPU_SSSE3) )
433         return;
434     pf[I_PRED_CHROMA_H]       = x264_predict_8x8c_h_ssse3;
435 #if HAVE_X86_INLINE_ASM
436     pf[I_PRED_CHROMA_P]       = x264_predict_8x8c_p_ssse3;
437     if( !(cpu&X264_CPU_AVX) )
438         return;
439     pf[I_PRED_CHROMA_P]       = x264_predict_8x8c_p_avx;
440 #endif
441 #endif // HIGH_BIT_DEPTH
442 }
443
444 void x264_predict_8x16c_init_mmx( int cpu, x264_predict_t pf[7] )
445 {
446     if( !(cpu&X264_CPU_MMX) )
447         return;
448 #if HIGH_BIT_DEPTH
449     if( !(cpu&X264_CPU_MMX2) )
450         return;
451     pf[I_PRED_CHROMA_DC]      = x264_predict_8x16c_dc_mmx2;
452     pf[I_PRED_CHROMA_H]       = x264_predict_8x16c_h_mmx2;
453     if( !(cpu&X264_CPU_SSE) )
454         return;
455     pf[I_PRED_CHROMA_V]       = x264_predict_8x16c_v_sse;
456     if( !(cpu&X264_CPU_SSE2) )
457         return;
458     pf[I_PRED_CHROMA_DC_TOP]  = x264_predict_8x16c_dc_top_sse2;
459     pf[I_PRED_CHROMA_DC]      = x264_predict_8x16c_dc_sse2;
460     pf[I_PRED_CHROMA_H]       = x264_predict_8x16c_h_sse2;
461     pf[I_PRED_CHROMA_P]       = x264_predict_8x16c_p_sse2;
462     if( !(cpu&X264_CPU_AVX) )
463         return;
464     pf[I_PRED_CHROMA_P]       = x264_predict_8x16c_p_avx;
465     if( !(cpu&X264_CPU_AVX2) )
466         return;
467     pf[I_PRED_CHROMA_H]   = x264_predict_8x16c_h_avx2;
468 #else
469     pf[I_PRED_CHROMA_V]       = x264_predict_8x16c_v_mmx;
470     if( !(cpu&X264_CPU_MMX2) )
471         return;
472     pf[I_PRED_CHROMA_DC_TOP]  = x264_predict_8x16c_dc_top_mmx2;
473     pf[I_PRED_CHROMA_DC]      = x264_predict_8x16c_dc_mmx2;
474     pf[I_PRED_CHROMA_H]       = x264_predict_8x16c_h_mmx2;
475 #ifndef ARCH_X86_64
476     pf[I_PRED_CHROMA_P]       = x264_predict_8x16c_p_mmx2;
477 #endif
478     if( !(cpu&X264_CPU_SSE2) )
479         return;
480     pf[I_PRED_CHROMA_P]       = x264_predict_8x16c_p_sse2;
481     if( !(cpu&X264_CPU_SSSE3) )
482         return;
483     pf[I_PRED_CHROMA_H]       = x264_predict_8x16c_h_ssse3;
484     if( !(cpu&X264_CPU_AVX) )
485         return;
486     pf[I_PRED_CHROMA_P]       = x264_predict_8x16c_p_avx;
487 #endif // HIGH_BIT_DEPTH
488 }
489
490 void x264_predict_8x8_init_mmx( int cpu, x264_predict8x8_t pf[12], x264_predict_8x8_filter_t *predict_8x8_filter )
491 {
492     if( !(cpu&X264_CPU_MMX2) )
493         return;
494 #if HIGH_BIT_DEPTH
495     if( !(cpu&X264_CPU_SSE) )
496         return;
497     pf[I_PRED_8x8_V]      = x264_predict_8x8_v_sse;
498     if( !(cpu&X264_CPU_SSE2) )
499         return;
500     pf[I_PRED_8x8_H]      = x264_predict_8x8_h_sse2;
501     pf[I_PRED_8x8_DC]     = x264_predict_8x8_dc_sse2;
502     pf[I_PRED_8x8_DC_TOP] = x264_predict_8x8_dc_top_sse2;
503     pf[I_PRED_8x8_DC_LEFT]= x264_predict_8x8_dc_left_sse2;
504     pf[I_PRED_8x8_DDL]    = x264_predict_8x8_ddl_sse2;
505     pf[I_PRED_8x8_DDR]    = x264_predict_8x8_ddr_sse2;
506     pf[I_PRED_8x8_VL]     = x264_predict_8x8_vl_sse2;
507     pf[I_PRED_8x8_VR]     = x264_predict_8x8_vr_sse2;
508     pf[I_PRED_8x8_HD]     = x264_predict_8x8_hd_sse2;
509     pf[I_PRED_8x8_HU]     = x264_predict_8x8_hu_sse2;
510     *predict_8x8_filter   = x264_predict_8x8_filter_sse2;
511     if( !(cpu&X264_CPU_SSSE3) )
512         return;
513     pf[I_PRED_8x8_DDL]    = x264_predict_8x8_ddl_ssse3;
514     pf[I_PRED_8x8_DDR]    = x264_predict_8x8_ddr_ssse3;
515     pf[I_PRED_8x8_HD]     = x264_predict_8x8_hd_ssse3;
516     pf[I_PRED_8x8_HU]     = x264_predict_8x8_hu_ssse3;
517     pf[I_PRED_8x8_VL]     = x264_predict_8x8_vl_ssse3;
518     pf[I_PRED_8x8_VR]     = x264_predict_8x8_vr_ssse3;
519     *predict_8x8_filter   = x264_predict_8x8_filter_ssse3;
520     if( cpu&X264_CPU_CACHELINE_64 )
521     {
522         pf[I_PRED_8x8_DDL]= x264_predict_8x8_ddl_ssse3_cache64;
523         pf[I_PRED_8x8_DDR]= x264_predict_8x8_ddr_ssse3_cache64;
524     }
525     if( !(cpu&X264_CPU_AVX) )
526         return;
527     pf[I_PRED_8x8_HD]     = x264_predict_8x8_hd_avx;
528     pf[I_PRED_8x8_HU]     = x264_predict_8x8_hu_avx;
529     pf[I_PRED_8x8_VL]     = x264_predict_8x8_vl_avx;
530     pf[I_PRED_8x8_VR]     = x264_predict_8x8_vr_avx;
531     *predict_8x8_filter   = x264_predict_8x8_filter_avx;
532 #else
533     pf[I_PRED_8x8_V]      = x264_predict_8x8_v_mmx2;
534     pf[I_PRED_8x8_H]      = x264_predict_8x8_h_mmx2;
535     pf[I_PRED_8x8_DC]     = x264_predict_8x8_dc_mmx2;
536     pf[I_PRED_8x8_DC_TOP] = x264_predict_8x8_dc_top_mmx2;
537     pf[I_PRED_8x8_DC_LEFT]= x264_predict_8x8_dc_left_mmx2;
538     pf[I_PRED_8x8_HD]     = x264_predict_8x8_hd_mmx2;
539     pf[I_PRED_8x8_VL]     = x264_predict_8x8_vl_mmx2;
540     *predict_8x8_filter   = x264_predict_8x8_filter_mmx2;
541 #if ARCH_X86
542     pf[I_PRED_8x8_DDL]  = x264_predict_8x8_ddl_mmx2;
543     pf[I_PRED_8x8_DDR]  = x264_predict_8x8_ddr_mmx2;
544     pf[I_PRED_8x8_VR]   = x264_predict_8x8_vr_mmx2;
545     pf[I_PRED_8x8_HU]   = x264_predict_8x8_hu_mmx2;
546 #endif
547     if( !(cpu&X264_CPU_SSE2) )
548         return;
549     pf[I_PRED_8x8_DDL]  = x264_predict_8x8_ddl_sse2;
550     pf[I_PRED_8x8_VL]   = x264_predict_8x8_vl_sse2;
551     pf[I_PRED_8x8_VR]   = x264_predict_8x8_vr_sse2;
552     pf[I_PRED_8x8_DDR]  = x264_predict_8x8_ddr_sse2;
553     pf[I_PRED_8x8_HD]   = x264_predict_8x8_hd_sse2;
554     pf[I_PRED_8x8_HU]   = x264_predict_8x8_hu_sse2;
555     if( !(cpu&X264_CPU_SSSE3) )
556         return;
557     if( !(cpu&X264_CPU_SLOW_PALIGNR) )
558     {
559         pf[I_PRED_8x8_DDL]  = x264_predict_8x8_ddl_ssse3;
560         pf[I_PRED_8x8_VR]   = x264_predict_8x8_vr_ssse3;
561     }
562     pf[I_PRED_8x8_HU]   = x264_predict_8x8_hu_ssse3;
563     *predict_8x8_filter = x264_predict_8x8_filter_ssse3;
564     if( !(cpu&X264_CPU_AVX) )
565         return;
566     pf[I_PRED_8x8_DDL]  = x264_predict_8x8_ddl_avx;
567     pf[I_PRED_8x8_DDR]  = x264_predict_8x8_ddr_avx;
568     pf[I_PRED_8x8_VL]   = x264_predict_8x8_vl_avx;
569     pf[I_PRED_8x8_VR]   = x264_predict_8x8_vr_avx;
570     pf[I_PRED_8x8_HD]   = x264_predict_8x8_hd_avx;
571 #endif // HIGH_BIT_DEPTH
572 }
573
574 void x264_predict_4x4_init_mmx( int cpu, x264_predict_t pf[12] )
575 {
576     if( !(cpu&X264_CPU_MMX2) )
577         return;
578     pf[I_PRED_4x4_DC]  = x264_predict_4x4_dc_mmx2;
579     pf[I_PRED_4x4_DDL] = x264_predict_4x4_ddl_mmx2;
580     pf[I_PRED_4x4_DDR] = x264_predict_4x4_ddr_mmx2;
581     pf[I_PRED_4x4_VL]  = x264_predict_4x4_vl_mmx2;
582     pf[I_PRED_4x4_HD]  = x264_predict_4x4_hd_mmx2;
583     pf[I_PRED_4x4_HU]  = x264_predict_4x4_hu_mmx2;
584 #if HIGH_BIT_DEPTH
585     if( !(cpu&X264_CPU_SSE2) )
586         return;
587     pf[I_PRED_4x4_DDL] = x264_predict_4x4_ddl_sse2;
588     pf[I_PRED_4x4_DDR] = x264_predict_4x4_ddr_sse2;
589     pf[I_PRED_4x4_HD]  = x264_predict_4x4_hd_sse2;
590     pf[I_PRED_4x4_VL]  = x264_predict_4x4_vl_sse2;
591     pf[I_PRED_4x4_VR]  = x264_predict_4x4_vr_sse2;
592     if( !(cpu&X264_CPU_SSSE3) )
593         return;
594     pf[I_PRED_4x4_DDR] = x264_predict_4x4_ddr_ssse3;
595     pf[I_PRED_4x4_VR]  = x264_predict_4x4_vr_ssse3;
596     pf[I_PRED_4x4_HD]  = x264_predict_4x4_hd_ssse3;
597     if( !(cpu&X264_CPU_AVX) )
598         return;
599     pf[I_PRED_4x4_DDL] = x264_predict_4x4_ddl_avx;
600     pf[I_PRED_4x4_DDR] = x264_predict_4x4_ddr_avx;
601     pf[I_PRED_4x4_HD]  = x264_predict_4x4_hd_avx;
602     pf[I_PRED_4x4_VL]  = x264_predict_4x4_vl_avx;
603     pf[I_PRED_4x4_VR]  = x264_predict_4x4_vr_avx;
604     if( !(cpu&X264_CPU_AVX2) )
605         return;
606     pf[I_PRED_4x4_H]  = x264_predict_4x4_h_avx2;
607 #else
608     pf[I_PRED_4x4_VR]  = x264_predict_4x4_vr_mmx2;
609     if( !(cpu&X264_CPU_SSSE3) )
610         return;
611     pf[I_PRED_4x4_DDR] = x264_predict_4x4_ddr_ssse3;
612     pf[I_PRED_4x4_VR]  = x264_predict_4x4_vr_ssse3;
613     pf[I_PRED_4x4_HD]  = x264_predict_4x4_hd_ssse3;
614     if( cpu&X264_CPU_CACHELINE_64 )
615         pf[I_PRED_4x4_VR] = x264_predict_4x4_vr_ssse3_cache64;
616 #endif // HIGH_BIT_DEPTH
617 }