9b0788ff73eb3c693101856d6728f86d18143907
[ffmpeg.git] / libavcodec / x86 / dsputil_init.c
1 /*
2  * This file is part of Libav.
3  *
4  * Libav is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * Libav is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with Libav; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include "config.h"
20 #include "libavutil/attributes.h"
21 #include "libavutil/cpu.h"
22 #include "libavutil/internal.h"
23 #include "libavutil/x86/cpu.h"
24 #include "libavcodec/avcodec.h"
25 #include "libavcodec/dsputil.h"
26 #include "libavcodec/pixels.h"
27 #include "libavcodec/simple_idct.h"
28 #include "libavcodec/version.h"
29 #include "dsputil_x86.h"
30 #include "fpel.h"
31 #include "idct_xvid.h"
32
33 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
34                               int dstStride, int src1Stride, int h);
35 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
36                                      uint8_t *src2, int dstStride,
37                                      int src1Stride, int h);
38 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
39                               int dstStride, int src1Stride, int h);
40 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
41                                int dstStride, int src1Stride, int h);
42 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
43                                int dstStride, int src1Stride, int h);
44 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
45                                       int dstStride, int src1Stride, int h);
46 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
47                                           int dstStride, int srcStride, int h);
48 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
49                                           int dstStride, int srcStride, int h);
50 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
51                                                  int dstStride, int srcStride,
52                                                  int h);
53 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
54                                          int dstStride, int srcStride, int h);
55 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
56                                          int dstStride, int srcStride, int h);
57 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
58                                                 int dstStride, int srcStride,
59                                                 int h);
60 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
61                                           int dstStride, int srcStride);
62 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
63                                           int dstStride, int srcStride);
64 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
65                                                  int dstStride, int srcStride);
66 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
67                                          int dstStride, int srcStride);
68 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
69                                          int dstStride, int srcStride);
70 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
71                                                 int dstStride, int srcStride);
72 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
73 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
74
75 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
76                                       int order);
77 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
78                                     int order);
79
80 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
81 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
82
83 void ff_vector_clip_int32_mmx(int32_t *dst, const int32_t *src,
84                               int32_t min, int32_t max, unsigned int len);
85 void ff_vector_clip_int32_sse2(int32_t *dst, const int32_t *src,
86                                int32_t min, int32_t max, unsigned int len);
87 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
88                                    int32_t min, int32_t max, unsigned int len);
89 void ff_vector_clip_int32_sse4(int32_t *dst, const int32_t *src,
90                                int32_t min, int32_t max, unsigned int len);
91
92 #if HAVE_YASM
93
94 CALL_2X_PIXELS(ff_avg_pixels16_mmxext, ff_avg_pixels8_mmxext, 8)
95 CALL_2X_PIXELS(ff_put_pixels16_mmxext, ff_put_pixels8_mmxext, 8)
96
97 #define QPEL_OP(OPNAME, RND, MMX)                                       \
98 static void OPNAME ## qpel8_mc00_ ## MMX(uint8_t *dst, uint8_t *src,    \
99                                          ptrdiff_t stride)              \
100 {                                                                       \
101     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
102 }                                                                       \
103                                                                         \
104 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
105                                          ptrdiff_t stride)              \
106 {                                                                       \
107     uint64_t temp[8];                                                   \
108     uint8_t *const half = (uint8_t *) temp;                             \
109     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
110                                                    stride, 8);          \
111     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
112                                         stride, stride, 8);             \
113 }                                                                       \
114                                                                         \
115 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
116                                          ptrdiff_t stride)              \
117 {                                                                       \
118     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
119                                                    stride, 8);          \
120 }                                                                       \
121                                                                         \
122 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
123                                          ptrdiff_t stride)              \
124 {                                                                       \
125     uint64_t temp[8];                                                   \
126     uint8_t *const half = (uint8_t *) temp;                             \
127     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
128                                                    stride, 8);          \
129     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
130                                         stride, 8);                     \
131 }                                                                       \
132                                                                         \
133 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
134                                          ptrdiff_t stride)              \
135 {                                                                       \
136     uint64_t temp[8];                                                   \
137     uint8_t *const half = (uint8_t *) temp;                             \
138     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
139                                                    8, stride);          \
140     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
141                                         stride, stride, 8);             \
142 }                                                                       \
143                                                                         \
144 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
145                                          ptrdiff_t stride)              \
146 {                                                                       \
147     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
148                                                    stride, stride);     \
149 }                                                                       \
150                                                                         \
151 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
152                                          ptrdiff_t stride)              \
153 {                                                                       \
154     uint64_t temp[8];                                                   \
155     uint8_t *const half = (uint8_t *) temp;                             \
156     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
157                                                    8, stride);          \
158     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
159                                         stride, 8);                     \
160 }                                                                       \
161                                                                         \
162 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
163                                          ptrdiff_t stride)              \
164 {                                                                       \
165     uint64_t half[8 + 9];                                               \
166     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
167     uint8_t *const halfHV = (uint8_t *) half;                           \
168     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
169                                                    stride, 9);          \
170     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
171                                         stride, 9);                     \
172     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
173     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
174                                         stride, 8, 8);                  \
175 }                                                                       \
176                                                                         \
177 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
178                                          ptrdiff_t stride)              \
179 {                                                                       \
180     uint64_t half[8 + 9];                                               \
181     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
182     uint8_t *const halfHV = (uint8_t *) half;                           \
183     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
184                                                    stride, 9);          \
185     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
186                                         stride, 9);                     \
187     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
188     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
189                                         stride, 8, 8);                  \
190 }                                                                       \
191                                                                         \
192 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
193                                          ptrdiff_t stride)              \
194 {                                                                       \
195     uint64_t half[8 + 9];                                               \
196     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
197     uint8_t *const halfHV = (uint8_t *) half;                           \
198     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
199                                                    stride, 9);          \
200     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
201                                         stride, 9);                     \
202     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
203     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
204                                         stride, 8, 8);                  \
205 }                                                                       \
206                                                                         \
207 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
208                                          ptrdiff_t stride)              \
209 {                                                                       \
210     uint64_t half[8 + 9];                                               \
211     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
212     uint8_t *const halfHV = (uint8_t *) half;                           \
213     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
214                                                    stride, 9);          \
215     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
216                                         stride, 9);                     \
217     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
218     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
219                                         stride, 8, 8);                  \
220 }                                                                       \
221                                                                         \
222 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
223                                          ptrdiff_t stride)              \
224 {                                                                       \
225     uint64_t half[8 + 9];                                               \
226     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
227     uint8_t *const halfHV = (uint8_t *) half;                           \
228     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
229                                                    stride, 9);          \
230     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
231     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
232                                         stride, 8, 8);                  \
233 }                                                                       \
234                                                                         \
235 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
236                                          ptrdiff_t stride)              \
237 {                                                                       \
238     uint64_t half[8 + 9];                                               \
239     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
240     uint8_t *const halfHV = (uint8_t *) half;                           \
241     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
242                                                    stride, 9);          \
243     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
244     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
245                                         stride, 8, 8);                  \
246 }                                                                       \
247                                                                         \
248 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
249                                          ptrdiff_t stride)              \
250 {                                                                       \
251     uint64_t half[8 + 9];                                               \
252     uint8_t *const halfH = (uint8_t *) half;                            \
253     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
254                                                    stride, 9);          \
255     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
256                                         8, stride, 9);                  \
257     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
258                                                    stride, 8);          \
259 }                                                                       \
260                                                                         \
261 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
262                                          ptrdiff_t stride)              \
263 {                                                                       \
264     uint64_t half[8 + 9];                                               \
265     uint8_t *const halfH = (uint8_t *) half;                            \
266     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
267                                                    stride, 9);          \
268     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
269                                         stride, 9);                     \
270     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
271                                                    stride, 8);          \
272 }                                                                       \
273                                                                         \
274 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
275                                          ptrdiff_t stride)              \
276 {                                                                       \
277     uint64_t half[9];                                                   \
278     uint8_t *const halfH = (uint8_t *) half;                            \
279     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
280                                                    stride, 9);          \
281     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
282                                                    stride, 8);          \
283 }                                                                       \
284                                                                         \
285 static void OPNAME ## qpel16_mc00_ ## MMX(uint8_t *dst, uint8_t *src,   \
286                                           ptrdiff_t stride)             \
287 {                                                                       \
288     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
289 }                                                                       \
290                                                                         \
291 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
292                                           ptrdiff_t stride)             \
293 {                                                                       \
294     uint64_t temp[32];                                                  \
295     uint8_t *const half = (uint8_t *) temp;                             \
296     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
297                                                     stride, 16);        \
298     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
299                                          stride, 16);                   \
300 }                                                                       \
301                                                                         \
302 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
303                                           ptrdiff_t stride)             \
304 {                                                                       \
305     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
306                                                     stride, stride, 16);\
307 }                                                                       \
308                                                                         \
309 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
310                                           ptrdiff_t stride)             \
311 {                                                                       \
312     uint64_t temp[32];                                                  \
313     uint8_t *const half = (uint8_t*) temp;                              \
314     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
315                                                     stride, 16);        \
316     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
317                                          stride, stride, 16);           \
318 }                                                                       \
319                                                                         \
320 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
321                                           ptrdiff_t stride)             \
322 {                                                                       \
323     uint64_t temp[32];                                                  \
324     uint8_t *const half = (uint8_t *) temp;                             \
325     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
326                                                     stride);            \
327     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
328                                          stride, 16);                   \
329 }                                                                       \
330                                                                         \
331 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
332                                           ptrdiff_t stride)             \
333 {                                                                       \
334     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
335                                                     stride, stride);    \
336 }                                                                       \
337                                                                         \
338 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
339                                           ptrdiff_t stride)             \
340 {                                                                       \
341     uint64_t temp[32];                                                  \
342     uint8_t *const half = (uint8_t *) temp;                             \
343     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
344                                                     stride);            \
345     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
346                                          stride, stride, 16);           \
347 }                                                                       \
348                                                                         \
349 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
350                                           ptrdiff_t stride)             \
351 {                                                                       \
352     uint64_t half[16 * 2 + 17 * 2];                                     \
353     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
354     uint8_t *const halfHV = (uint8_t *) half;                           \
355     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
356                                                     stride, 17);        \
357     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
358                                          stride, 17);                   \
359     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
360                                                     16, 16);            \
361     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
362                                          stride, 16, 16);               \
363 }                                                                       \
364                                                                         \
365 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
366                                           ptrdiff_t stride)             \
367 {                                                                       \
368     uint64_t half[16 * 2 + 17 * 2];                                     \
369     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
370     uint8_t *const halfHV = (uint8_t *) half;                           \
371     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
372                                                     stride, 17);        \
373     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
374                                          stride, 17);                   \
375     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
376                                                     16, 16);            \
377     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
378                                          stride, 16, 16);               \
379 }                                                                       \
380                                                                         \
381 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
382                                           ptrdiff_t stride)             \
383 {                                                                       \
384     uint64_t half[16 * 2 + 17 * 2];                                     \
385     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
386     uint8_t *const halfHV = (uint8_t *) half;                           \
387     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
388                                                     stride, 17);        \
389     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
390                                          stride, 17);                   \
391     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
392                                                     16, 16);            \
393     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
394                                          stride, 16, 16);               \
395 }                                                                       \
396                                                                         \
397 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
398                                           ptrdiff_t stride)             \
399 {                                                                       \
400     uint64_t half[16 * 2 + 17 * 2];                                     \
401     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
402     uint8_t *const halfHV = (uint8_t *) half;                           \
403     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
404                                                     stride, 17);        \
405     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
406                                          stride, 17);                   \
407     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
408                                                     16, 16);            \
409     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
410                                          stride, 16, 16);               \
411 }                                                                       \
412                                                                         \
413 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
414                                           ptrdiff_t stride)             \
415 {                                                                       \
416     uint64_t half[16 * 2 + 17 * 2];                                     \
417     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
418     uint8_t *const halfHV = (uint8_t *) half;                           \
419     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
420                                                     stride, 17);        \
421     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
422                                                     16, 16);            \
423     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
424                                          stride, 16, 16);               \
425 }                                                                       \
426                                                                         \
427 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
428                                           ptrdiff_t stride)             \
429 {                                                                       \
430     uint64_t half[16 * 2 + 17 * 2];                                     \
431     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
432     uint8_t *const halfHV = (uint8_t *) half;                           \
433     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
434                                                     stride, 17);        \
435     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
436                                                     16, 16);            \
437     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
438                                          stride, 16, 16);               \
439 }                                                                       \
440                                                                         \
441 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
442                                           ptrdiff_t stride)             \
443 {                                                                       \
444     uint64_t half[17 * 2];                                              \
445     uint8_t *const halfH = (uint8_t *) half;                            \
446     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
447                                                     stride, 17);        \
448     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
449                                          stride, 17);                   \
450     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
451                                                     stride, 16);        \
452 }                                                                       \
453                                                                         \
454 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
455                                           ptrdiff_t stride)             \
456 {                                                                       \
457     uint64_t half[17 * 2];                                              \
458     uint8_t *const halfH = (uint8_t *) half;                            \
459     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
460                                                     stride, 17);        \
461     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
462                                          stride, 17);                   \
463     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
464                                                     stride, 16);        \
465 }                                                                       \
466                                                                         \
467 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
468                                           ptrdiff_t stride)             \
469 {                                                                       \
470     uint64_t half[17 * 2];                                              \
471     uint8_t *const halfH = (uint8_t *) half;                            \
472     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
473                                                     stride, 17);        \
474     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
475                                                     stride, 16);        \
476 }
477
478 QPEL_OP(put_,        _,        mmxext)
479 QPEL_OP(avg_,        _,        mmxext)
480 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
481
482 #endif /* HAVE_YASM */
483
484 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
485 do {                                                                         \
486     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
487     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
488     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
489     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
490     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
491     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
492     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
493     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
494     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
495     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
496     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
497     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
498     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
499     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
500     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
501     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
502 } while (0)
503
504 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
505                                      int cpu_flags, unsigned high_bit_depth)
506 {
507 #if HAVE_MMX_INLINE
508     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
509     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
510     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
511
512     if (!high_bit_depth) {
513         c->clear_block  = ff_clear_block_mmx;
514         c->clear_blocks = ff_clear_blocks_mmx;
515         c->draw_edges   = ff_draw_edges_mmx;
516
517         switch (avctx->idct_algo) {
518         case FF_IDCT_AUTO:
519         case FF_IDCT_SIMPLEMMX:
520             c->idct_put              = ff_simple_idct_put_mmx;
521             c->idct_add              = ff_simple_idct_add_mmx;
522             c->idct                  = ff_simple_idct_mmx;
523             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
524             break;
525         case FF_IDCT_XVIDMMX:
526             c->idct_put              = ff_idct_xvid_mmx_put;
527             c->idct_add              = ff_idct_xvid_mmx_add;
528             c->idct                  = ff_idct_xvid_mmx;
529             break;
530         }
531     }
532
533     c->gmc = ff_gmc_mmx;
534 #endif /* HAVE_MMX_INLINE */
535
536 #if HAVE_MMX_EXTERNAL
537     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
538 #endif /* HAVE_MMX_EXTERNAL */
539 }
540
541 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
542                                         int cpu_flags, unsigned high_bit_depth)
543 {
544 #if HAVE_MMXEXT_INLINE
545     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
546         c->idct_put = ff_idct_xvid_mmxext_put;
547         c->idct_add = ff_idct_xvid_mmxext_add;
548         c->idct     = ff_idct_xvid_mmxext;
549     }
550 #endif /* HAVE_MMXEXT_INLINE */
551
552 #if HAVE_MMXEXT_EXTERNAL
553     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
554     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
555
556     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
557     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
558     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
559     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
560
561     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
562 #endif /* HAVE_MMXEXT_EXTERNAL */
563 }
564
565 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
566                                      int cpu_flags, unsigned high_bit_depth)
567 {
568 #if HAVE_SSE_INLINE
569     c->vector_clipf = ff_vector_clipf_sse;
570
571 #if FF_API_XVMC
572 FF_DISABLE_DEPRECATION_WARNINGS
573     /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
574     if (CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)
575         return;
576 FF_ENABLE_DEPRECATION_WARNINGS
577 #endif /* FF_API_XVMC */
578
579     if (!high_bit_depth) {
580         c->clear_block  = ff_clear_block_sse;
581         c->clear_blocks = ff_clear_blocks_sse;
582     }
583 #endif /* HAVE_SSE_INLINE */
584 }
585
586 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
587                                       int cpu_flags, unsigned high_bit_depth)
588 {
589 #if HAVE_SSE2_INLINE
590     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
591         c->idct_put              = ff_idct_xvid_sse2_put;
592         c->idct_add              = ff_idct_xvid_sse2_add;
593         c->idct                  = ff_idct_xvid_sse2;
594         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
595     }
596 #endif /* HAVE_SSE2_INLINE */
597
598 #if HAVE_SSE2_EXTERNAL
599     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
600     if (cpu_flags & AV_CPU_FLAG_ATOM) {
601         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
602     } else {
603         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
604     }
605     c->bswap_buf = ff_bswap32_buf_sse2;
606 #endif /* HAVE_SSE2_EXTERNAL */
607 }
608
609 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
610                                        int cpu_flags, unsigned high_bit_depth)
611 {
612 #if HAVE_SSSE3_EXTERNAL
613     c->bswap_buf = ff_bswap32_buf_ssse3;
614 #endif /* HAVE_SSSE3_EXTERNAL */
615 }
616
617 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
618                                       int cpu_flags, unsigned high_bit_depth)
619 {
620 #if HAVE_SSE4_EXTERNAL
621     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
622 #endif /* HAVE_SSE4_EXTERNAL */
623 }
624
625 av_cold void ff_dsputil_init_x86(DSPContext *c, AVCodecContext *avctx,
626                                  unsigned high_bit_depth)
627 {
628     int cpu_flags = av_get_cpu_flags();
629
630     if (X86_MMX(cpu_flags))
631         dsputil_init_mmx(c, avctx, cpu_flags, high_bit_depth);
632
633     if (X86_MMXEXT(cpu_flags))
634         dsputil_init_mmxext(c, avctx, cpu_flags, high_bit_depth);
635
636     if (X86_SSE(cpu_flags))
637         dsputil_init_sse(c, avctx, cpu_flags, high_bit_depth);
638
639     if (X86_SSE2(cpu_flags))
640         dsputil_init_sse2(c, avctx, cpu_flags, high_bit_depth);
641
642     if (EXTERNAL_SSSE3(cpu_flags))
643         dsputil_init_ssse3(c, avctx, cpu_flags, high_bit_depth);
644
645     if (EXTERNAL_SSE4(cpu_flags))
646         dsputil_init_sse4(c, avctx, cpu_flags, high_bit_depth);
647
648     if (CONFIG_ENCODERS)
649         ff_dsputilenc_init_mmx(c, avctx, high_bit_depth);
650 }