x86: dsputil: Simplify xvmc deprecation conditional
[ffmpeg.git] / libavcodec / x86 / dsputil_init.c
1 /*
2  * This file is part of Libav.
3  *
4  * Libav is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * Libav is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with Libav; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include "config.h"
20 #include "libavutil/attributes.h"
21 #include "libavutil/cpu.h"
22 #include "libavutil/internal.h"
23 #include "libavutil/x86/asm.h"
24 #include "libavutil/x86/cpu.h"
25 #include "libavcodec/dsputil.h"
26 #include "libavcodec/simple_idct.h"
27 #include "libavcodec/version.h"
28 #include "dsputil_x86.h"
29 #include "idct_xvid.h"
30
31 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
32                               int dstStride, int src1Stride, int h);
33 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
34                                      uint8_t *src2, int dstStride,
35                                      int src1Stride, int h);
36 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
37                               int dstStride, int src1Stride, int h);
38 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
39                                int dstStride, int src1Stride, int h);
40 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
41                                int dstStride, int src1Stride, int h);
42 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
43                                       int dstStride, int src1Stride, int h);
44 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
45                                          int dstStride, int srcStride, int h);
46 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
47                                          int dstStride, int srcStride, int h);
48 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
49                                                  int dstStride, int srcStride,
50                                                  int h);
51 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
52                                         int dstStride, int srcStride, int h);
53 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
54                                         int dstStride, int srcStride, int h);
55 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
56                                                 int dstStride, int srcStride,
57                                                 int h);
58 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
59                                          int dstStride, int srcStride);
60 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
61                                          int dstStride, int srcStride);
62 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
63                                                  int dstStride, int srcStride);
64 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
65                                         int dstStride, int srcStride);
66 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
67                                         int dstStride, int srcStride);
68 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
69                                                 int dstStride, int srcStride);
70 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
71 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
72
73 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
74                                       int order);
75 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
76                                     int order);
77 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
78                                                const int16_t *v3,
79                                                int order, int mul);
80 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
81                                              const int16_t *v3,
82                                              int order, int mul);
83 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
84                                               const int16_t *v3,
85                                               int order, int mul);
86
87 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
88 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
89
90 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
91                                           const uint8_t *diff, int w,
92                                           int *left, int *left_top);
93 int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
94                                        int w, int left);
95 int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
96                                       int w, int left);
97
98 void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,
99                                    int32_t min, int32_t max, unsigned int len);
100 void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,
101                                    int32_t min, int32_t max, unsigned int len);
102 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
103                                    int32_t min, int32_t max, unsigned int len);
104 void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,
105                                    int32_t min, int32_t max, unsigned int len);
106
107 #if HAVE_YASM
108
109 PIXELS16(static, ff_avg, , , _mmxext)
110 PIXELS16(static, ff_put, , , _mmxext)
111
112 #define QPEL_OP(OPNAME, RND, MMX)                                       \
113 static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \
114                                           ptrdiff_t stride)             \
115 {                                                                       \
116     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
117 }                                                                       \
118                                                                         \
119 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
120                                          ptrdiff_t stride)              \
121 {                                                                       \
122     uint64_t temp[8];                                                   \
123     uint8_t * const half = (uint8_t*)temp;                              \
124     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
125                                                    stride, 8);          \
126     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
127                                         stride, stride, 8);             \
128 }                                                                       \
129                                                                         \
130 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
131                                          ptrdiff_t stride)              \
132 {                                                                       \
133     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
134                                                    stride, 8);          \
135 }                                                                       \
136                                                                         \
137 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
138                                          ptrdiff_t stride)              \
139 {                                                                       \
140     uint64_t temp[8];                                                   \
141     uint8_t * const half = (uint8_t*)temp;                              \
142     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
143                                                    stride, 8);          \
144     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
145                                         stride, 8);                     \
146 }                                                                       \
147                                                                         \
148 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
149                                          ptrdiff_t stride)              \
150 {                                                                       \
151     uint64_t temp[8];                                                   \
152     uint8_t * const half = (uint8_t*)temp;                              \
153     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
154                                                    8, stride);          \
155     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
156                                         stride, stride, 8);             \
157 }                                                                       \
158                                                                         \
159 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
160                                          ptrdiff_t stride)              \
161 {                                                                       \
162     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
163                                                    stride, stride);     \
164 }                                                                       \
165                                                                         \
166 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
167                                          ptrdiff_t stride)              \
168 {                                                                       \
169     uint64_t temp[8];                                                   \
170     uint8_t * const half = (uint8_t*)temp;                              \
171     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
172                                                    8, stride);          \
173     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
174                                         stride, 8);                     \
175 }                                                                       \
176                                                                         \
177 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
178                                          ptrdiff_t stride)              \
179 {                                                                       \
180     uint64_t half[8 + 9];                                               \
181     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
182     uint8_t * const halfHV = ((uint8_t*)half);                          \
183     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
184                                                    stride, 9);          \
185     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
186                                         stride, 9);                     \
187     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
188     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
189                                         stride, 8, 8);                  \
190 }                                                                       \
191                                                                         \
192 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
193                                          ptrdiff_t stride)              \
194 {                                                                       \
195     uint64_t half[8 + 9];                                               \
196     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
197     uint8_t * const halfHV = ((uint8_t*)half);                          \
198     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
199                                                    stride, 9);          \
200     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
201                                         stride, 9);                     \
202     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
203     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
204                                         stride, 8, 8);                  \
205 }                                                                       \
206                                                                         \
207 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
208                                          ptrdiff_t stride)              \
209 {                                                                       \
210     uint64_t half[8 + 9];                                               \
211     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
212     uint8_t * const halfHV = ((uint8_t*)half);                          \
213     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
214                                                    stride, 9);          \
215     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
216                                         stride, 9);                     \
217     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
218     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
219                                         stride, 8, 8);                  \
220 }                                                                       \
221                                                                         \
222 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
223                                          ptrdiff_t stride)              \
224 {                                                                       \
225     uint64_t half[8 + 9];                                               \
226     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
227     uint8_t * const halfHV = ((uint8_t*)half);                          \
228     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
229                                                    stride, 9);          \
230     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
231                                         stride, 9);                     \
232     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
233     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
234                                         stride, 8, 8);                  \
235 }                                                                       \
236                                                                         \
237 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
238                                          ptrdiff_t stride)              \
239 {                                                                       \
240     uint64_t half[8 + 9];                                               \
241     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
242     uint8_t * const halfHV = ((uint8_t*)half);                          \
243     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
244                                                    stride, 9);          \
245     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
246     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
247                                         stride, 8, 8);                  \
248 }                                                                       \
249                                                                         \
250 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
251                                          ptrdiff_t stride)              \
252 {                                                                       \
253     uint64_t half[8 + 9];                                               \
254     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
255     uint8_t * const halfHV = ((uint8_t*)half);                          \
256     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
257                                                    stride, 9);          \
258     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
259     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
260                                         stride, 8, 8);                  \
261 }                                                                       \
262                                                                         \
263 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
264                                          ptrdiff_t stride)              \
265 {                                                                       \
266     uint64_t half[8 + 9];                                               \
267     uint8_t * const halfH = ((uint8_t*)half);                           \
268     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
269                                                    stride, 9);          \
270     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
271                                         8, stride, 9);                  \
272     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
273                                                    stride, 8);          \
274 }                                                                       \
275                                                                         \
276 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
277                                          ptrdiff_t stride)              \
278 {                                                                       \
279     uint64_t half[8 + 9];                                               \
280     uint8_t * const halfH = ((uint8_t*)half);                           \
281     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
282                                                    stride, 9);          \
283     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
284                                         stride, 9);                     \
285     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
286                                                    stride, 8);          \
287 }                                                                       \
288                                                                         \
289 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
290                                          ptrdiff_t stride)              \
291 {                                                                       \
292     uint64_t half[9];                                                   \
293     uint8_t * const halfH = ((uint8_t*)half);                           \
294     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
295                                                    stride, 9);          \
296     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
297                                                    stride, 8);          \
298 }                                                                       \
299                                                                         \
300 static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \
301                                            ptrdiff_t stride)            \
302 {                                                                       \
303     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
304 }                                                                       \
305                                                                         \
306 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
307                                           ptrdiff_t stride)             \
308 {                                                                       \
309     uint64_t temp[32];                                                  \
310     uint8_t * const half = (uint8_t*)temp;                              \
311     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
312                                                     stride, 16);        \
313     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
314                                          stride, 16);                   \
315 }                                                                       \
316                                                                         \
317 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
318                                           ptrdiff_t stride)             \
319 {                                                                       \
320     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
321                                                     stride, stride, 16);\
322 }                                                                       \
323                                                                         \
324 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
325                                           ptrdiff_t stride)             \
326 {                                                                       \
327     uint64_t temp[32];                                                  \
328     uint8_t * const half = (uint8_t*)temp;                              \
329     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
330                                                     stride, 16);        \
331     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
332                                          stride, stride, 16);           \
333 }                                                                       \
334                                                                         \
335 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
336                                           ptrdiff_t stride)             \
337 {                                                                       \
338     uint64_t temp[32];                                                  \
339     uint8_t * const half = (uint8_t*)temp;                              \
340     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
341                                                     stride);            \
342     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
343                                          stride, 16);                   \
344 }                                                                       \
345                                                                         \
346 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
347                                           ptrdiff_t stride)             \
348 {                                                                       \
349     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
350                                                     stride, stride);    \
351 }                                                                       \
352                                                                         \
353 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
354                                           ptrdiff_t stride)             \
355 {                                                                       \
356     uint64_t temp[32];                                                  \
357     uint8_t * const half = (uint8_t*)temp;                              \
358     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
359                                                     stride);            \
360     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
361                                          stride, stride, 16);           \
362 }                                                                       \
363                                                                         \
364 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
365                                           ptrdiff_t stride)             \
366 {                                                                       \
367     uint64_t half[16 * 2 + 17 * 2];                                     \
368     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
369     uint8_t * const halfHV = ((uint8_t*)half);                          \
370     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
371                                                     stride, 17);        \
372     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
373                                          stride, 17);                   \
374     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
375                                                     16, 16);            \
376     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
377                                          stride, 16, 16);               \
378 }                                                                       \
379                                                                         \
380 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
381                                           ptrdiff_t stride)             \
382 {                                                                       \
383     uint64_t half[16 * 2 + 17 * 2];                                     \
384     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
385     uint8_t * const halfHV = ((uint8_t*)half);                          \
386     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
387                                                     stride, 17);        \
388     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
389                                          stride, 17);                   \
390     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
391                                                     16, 16);            \
392     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
393                                          stride, 16, 16);               \
394 }                                                                       \
395                                                                         \
396 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
397                                           ptrdiff_t stride)             \
398 {                                                                       \
399     uint64_t half[16 * 2 + 17 * 2];                                     \
400     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
401     uint8_t * const halfHV = ((uint8_t*)half);                          \
402     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
403                                                     stride, 17);        \
404     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
405                                          stride, 17);                   \
406     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
407                                                     16, 16);            \
408     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
409                                          stride, 16, 16);               \
410 }                                                                       \
411                                                                         \
412 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
413                                           ptrdiff_t stride)             \
414 {                                                                       \
415     uint64_t half[16 * 2 + 17 * 2];                                     \
416     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
417     uint8_t * const halfHV = ((uint8_t*)half);                          \
418     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
419                                                     stride, 17);        \
420     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
421                                          stride, 17);                   \
422     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
423                                                     16, 16);            \
424     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
425                                          stride, 16, 16);               \
426 }                                                                       \
427                                                                         \
428 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
429                                           ptrdiff_t stride)             \
430 {                                                                       \
431     uint64_t half[16 * 2 + 17 * 2];                                     \
432     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
433     uint8_t * const halfHV = ((uint8_t*)half);                          \
434     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
435                                                     stride, 17);        \
436     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
437                                                     16, 16);            \
438     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
439                                          stride, 16, 16);               \
440 }                                                                       \
441                                                                         \
442 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
443                                           ptrdiff_t stride)             \
444 {                                                                       \
445     uint64_t half[16 * 2 + 17 * 2];                                     \
446     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
447     uint8_t * const halfHV = ((uint8_t*)half);                          \
448     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
449                                                     stride, 17);        \
450     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
451                                                     16, 16);            \
452     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
453                                          stride, 16, 16);               \
454 }                                                                       \
455                                                                         \
456 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
457                                           ptrdiff_t stride)             \
458 {                                                                       \
459     uint64_t half[17 * 2];                                              \
460     uint8_t * const halfH = ((uint8_t*)half);                           \
461     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
462                                                     stride, 17);        \
463     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
464                                          stride, 17);                   \
465     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
466                                                     stride, 16);        \
467 }                                                                       \
468                                                                         \
469 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
470                                           ptrdiff_t stride)             \
471 {                                                                       \
472     uint64_t half[17 * 2];                                              \
473     uint8_t * const halfH = ((uint8_t*)half);                           \
474     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
475                                                     stride, 17);        \
476     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
477                                          stride, 17);                   \
478     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
479                                                     stride, 16);        \
480 }                                                                       \
481                                                                         \
482 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
483                                           ptrdiff_t stride)             \
484 {                                                                       \
485     uint64_t half[17 * 2];                                              \
486     uint8_t * const halfH = ((uint8_t*)half);                           \
487     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
488                                                     stride, 17);        \
489     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
490                                                     stride, 16);        \
491 }
492
493 QPEL_OP(put_,        _,        mmxext)
494 QPEL_OP(avg_,        _,        mmxext)
495 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
496 #endif /* HAVE_YASM */
497
498 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
499     do {                                                                     \
500     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
501     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
502     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
503     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
504     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
505     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
506     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
507     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
508     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
509     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
510     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
511     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
512     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
513     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
514     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
515     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
516     } while (0)
517
518 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
519                                      int cpu_flags)
520 {
521 #if HAVE_MMX_INLINE
522     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
523
524     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
525     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
526     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
527
528     if (!high_bit_depth) {
529         c->clear_block  = ff_clear_block_mmx;
530         c->clear_blocks = ff_clear_blocks_mmx;
531         c->draw_edges   = ff_draw_edges_mmx;
532
533         switch (avctx->idct_algo) {
534         case FF_IDCT_AUTO:
535         case FF_IDCT_SIMPLEMMX:
536             c->idct_put              = ff_simple_idct_put_mmx;
537             c->idct_add              = ff_simple_idct_add_mmx;
538             c->idct                  = ff_simple_idct_mmx;
539             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
540             break;
541         case FF_IDCT_XVIDMMX:
542             c->idct_put              = ff_idct_xvid_mmx_put;
543             c->idct_add              = ff_idct_xvid_mmx_add;
544             c->idct                  = ff_idct_xvid_mmx;
545             break;
546         }
547     }
548
549     c->gmc = ff_gmc_mmx;
550
551     c->add_bytes = ff_add_bytes_mmx;
552 #endif /* HAVE_MMX_INLINE */
553
554 #if HAVE_MMX_EXTERNAL
555     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
556 #endif /* HAVE_MMX_EXTERNAL */
557 }
558
559 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
560                                         int cpu_flags)
561 {
562 #if HAVE_MMXEXT_INLINE
563     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
564
565     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
566         c->idct_put = ff_idct_xvid_mmxext_put;
567         c->idct_add = ff_idct_xvid_mmxext_add;
568         c->idct     = ff_idct_xvid_mmxext;
569     }
570 #endif /* HAVE_MMXEXT_INLINE */
571
572 #if HAVE_MMXEXT_EXTERNAL
573     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
574     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
575
576     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
577     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
578     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
579     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
580
581     /* slower than cmov version on AMD */
582     if (!(cpu_flags & AV_CPU_FLAG_3DNOW))
583         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
584
585     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
586     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
587 #endif /* HAVE_MMXEXT_EXTERNAL */
588 }
589
590 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
591                                      int cpu_flags)
592 {
593 #if HAVE_SSE_INLINE
594     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
595
596     c->vector_clipf = ff_vector_clipf_sse;
597
598 #if FF_API_XVMC
599 FF_DISABLE_DEPRECATION_WARNINGS
600     /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
601     if (CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)
602         return;
603 FF_ENABLE_DEPRECATION_WARNINGS
604 #endif /* FF_API_XVMC */
605
606     if (!high_bit_depth) {
607         c->clear_block  = ff_clear_block_sse;
608         c->clear_blocks = ff_clear_blocks_sse;
609     }
610 #endif /* HAVE_SSE_INLINE */
611 }
612
613 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
614                                       int cpu_flags)
615 {
616 #if HAVE_SSE2_INLINE
617     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
618
619     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
620         c->idct_put              = ff_idct_xvid_sse2_put;
621         c->idct_add              = ff_idct_xvid_sse2_add;
622         c->idct                  = ff_idct_xvid_sse2;
623         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
624     }
625 #endif /* HAVE_SSE2_INLINE */
626
627 #if HAVE_SSE2_EXTERNAL
628     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
629     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
630     if (cpu_flags & AV_CPU_FLAG_ATOM) {
631         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
632     } else {
633         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
634     }
635     c->bswap_buf = ff_bswap32_buf_sse2;
636 #endif /* HAVE_SSE2_EXTERNAL */
637 }
638
639 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
640                                        int cpu_flags)
641 {
642 #if HAVE_SSSE3_EXTERNAL
643     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
644     if (cpu_flags & AV_CPU_FLAG_SSE4) // not really SSE4, just slow on Conroe
645         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
646
647     if (!(cpu_flags & (AV_CPU_FLAG_SSE42 | AV_CPU_FLAG_3DNOW))) // cachesplit
648         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
649     c->bswap_buf = ff_bswap32_buf_ssse3;
650 #endif /* HAVE_SSSE3_EXTERNAL */
651 }
652
653 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
654                                       int cpu_flags)
655 {
656 #if HAVE_SSE4_EXTERNAL
657     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
658 #endif /* HAVE_SSE4_EXTERNAL */
659 }
660
661 av_cold void ff_dsputil_init_x86(DSPContext *c, AVCodecContext *avctx)
662 {
663     int cpu_flags = av_get_cpu_flags();
664
665 #if HAVE_7REGS && HAVE_INLINE_ASM
666     if (cpu_flags & AV_CPU_FLAG_CMOV)
667         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_cmov;
668 #endif
669
670     if (X86_MMX(cpu_flags))
671         dsputil_init_mmx(c, avctx, cpu_flags);
672
673     if (X86_MMXEXT(cpu_flags))
674         dsputil_init_mmxext(c, avctx, cpu_flags);
675
676     if (X86_SSE(cpu_flags))
677         dsputil_init_sse(c, avctx, cpu_flags);
678
679     if (X86_SSE2(cpu_flags))
680         dsputil_init_sse2(c, avctx, cpu_flags);
681
682     if (EXTERNAL_SSSE3(cpu_flags))
683         dsputil_init_ssse3(c, avctx, cpu_flags);
684
685     if (EXTERNAL_SSE4(cpu_flags))
686         dsputil_init_sse4(c, avctx, cpu_flags);
687
688     if (CONFIG_ENCODERS)
689         ff_dsputilenc_init_mmx(c, avctx);
690 }