x86: dsputil: Rename dsputil_mmx.h --> dsputil_x86.h
[ffmpeg.git] / libavcodec / x86 / dsputil_init.c
1 /*
2  * This file is part of Libav.
3  *
4  * Libav is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * Libav is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with Libav; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include "config.h"
20 #include "libavutil/attributes.h"
21 #include "libavutil/cpu.h"
22 #include "libavutil/x86/asm.h"
23 #include "libavcodec/dsputil.h"
24 #include "libavcodec/simple_idct.h"
25 #include "dsputil_x86.h"
26 #include "idct_xvid.h"
27
28 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
29                               int dstStride, int src1Stride, int h);
30 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
31                                      uint8_t *src2, int dstStride,
32                                      int src1Stride, int h);
33 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
34                               int dstStride, int src1Stride, int h);
35 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
36                                int dstStride, int src1Stride, int h);
37 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
38                                int dstStride, int src1Stride, int h);
39 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
40                                       int dstStride, int src1Stride, int h);
41 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
42                                          int dstStride, int srcStride, int h);
43 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
44                                          int dstStride, int srcStride, int h);
45 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
46                                                  int dstStride, int srcStride,
47                                                  int h);
48 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
49                                         int dstStride, int srcStride, int h);
50 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
51                                         int dstStride, int srcStride, int h);
52 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
53                                                 int dstStride, int srcStride,
54                                                 int h);
55 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
56                                          int dstStride, int srcStride);
57 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
58                                          int dstStride, int srcStride);
59 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
60                                                  int dstStride, int srcStride);
61 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
62                                         int dstStride, int srcStride);
63 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
64                                         int dstStride, int srcStride);
65 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
66                                                 int dstStride, int srcStride);
67 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
68 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
69
70 void ff_h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale);
71 void ff_h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale);
72
73 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
74                                       int order);
75 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
76                                     int order);
77 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
78                                                const int16_t *v3,
79                                                int order, int mul);
80 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
81                                              const int16_t *v3,
82                                              int order, int mul);
83 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
84                                               const int16_t *v3,
85                                               int order, int mul);
86
87 void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,
88                                         const int16_t *window, unsigned int len);
89 void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,
90                                       const int16_t *window, unsigned int len);
91 void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,
92                                   const int16_t *window, unsigned int len);
93 void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,
94                                 const int16_t *window, unsigned int len);
95 void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,
96                                  const int16_t *window, unsigned int len);
97 void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,
98                                       const int16_t *window, unsigned int len);
99
100 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
101 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
102
103 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
104                                           const uint8_t *diff, int w,
105                                           int *left, int *left_top);
106 int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
107                                        int w, int left);
108 int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
109                                       int w, int left);
110
111 void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,
112                                    int32_t min, int32_t max, unsigned int len);
113 void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,
114                                    int32_t min, int32_t max, unsigned int len);
115 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
116                                    int32_t min, int32_t max, unsigned int len);
117 void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,
118                                    int32_t min, int32_t max, unsigned int len);
119
120 #if HAVE_YASM
121
122 PIXELS16(static, ff_avg, , , _mmxext)
123 PIXELS16(static, ff_put, , , _mmxext)
124
125 #define QPEL_OP(OPNAME, RND, MMX)                                       \
126 static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \
127                                           ptrdiff_t stride)             \
128 {                                                                       \
129     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
130 }                                                                       \
131                                                                         \
132 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
133                                          ptrdiff_t stride)              \
134 {                                                                       \
135     uint64_t temp[8];                                                   \
136     uint8_t * const half = (uint8_t*)temp;                              \
137     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
138                                                    stride, 8);          \
139     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
140                                         stride, stride, 8);             \
141 }                                                                       \
142                                                                         \
143 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
144                                          ptrdiff_t stride)              \
145 {                                                                       \
146     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
147                                                    stride, 8);          \
148 }                                                                       \
149                                                                         \
150 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
151                                          ptrdiff_t stride)              \
152 {                                                                       \
153     uint64_t temp[8];                                                   \
154     uint8_t * const half = (uint8_t*)temp;                              \
155     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
156                                                    stride, 8);          \
157     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
158                                         stride, 8);                     \
159 }                                                                       \
160                                                                         \
161 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
162                                          ptrdiff_t stride)              \
163 {                                                                       \
164     uint64_t temp[8];                                                   \
165     uint8_t * const half = (uint8_t*)temp;                              \
166     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
167                                                    8, stride);          \
168     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
169                                         stride, stride, 8);             \
170 }                                                                       \
171                                                                         \
172 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
173                                          ptrdiff_t stride)              \
174 {                                                                       \
175     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
176                                                    stride, stride);     \
177 }                                                                       \
178                                                                         \
179 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
180                                          ptrdiff_t stride)              \
181 {                                                                       \
182     uint64_t temp[8];                                                   \
183     uint8_t * const half = (uint8_t*)temp;                              \
184     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
185                                                    8, stride);          \
186     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
187                                         stride, 8);                     \
188 }                                                                       \
189                                                                         \
190 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
191                                          ptrdiff_t stride)              \
192 {                                                                       \
193     uint64_t half[8 + 9];                                               \
194     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
195     uint8_t * const halfHV = ((uint8_t*)half);                          \
196     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
197                                                    stride, 9);          \
198     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
199                                         stride, 9);                     \
200     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
201     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
202                                         stride, 8, 8);                  \
203 }                                                                       \
204                                                                         \
205 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
206                                          ptrdiff_t stride)              \
207 {                                                                       \
208     uint64_t half[8 + 9];                                               \
209     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
210     uint8_t * const halfHV = ((uint8_t*)half);                          \
211     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
212                                                    stride, 9);          \
213     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
214                                         stride, 9);                     \
215     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
216     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
217                                         stride, 8, 8);                  \
218 }                                                                       \
219                                                                         \
220 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
221                                          ptrdiff_t stride)              \
222 {                                                                       \
223     uint64_t half[8 + 9];                                               \
224     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
225     uint8_t * const halfHV = ((uint8_t*)half);                          \
226     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
227                                                    stride, 9);          \
228     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
229                                         stride, 9);                     \
230     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
231     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
232                                         stride, 8, 8);                  \
233 }                                                                       \
234                                                                         \
235 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
236                                          ptrdiff_t stride)              \
237 {                                                                       \
238     uint64_t half[8 + 9];                                               \
239     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
240     uint8_t * const halfHV = ((uint8_t*)half);                          \
241     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
242                                                    stride, 9);          \
243     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
244                                         stride, 9);                     \
245     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
246     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
247                                         stride, 8, 8);                  \
248 }                                                                       \
249                                                                         \
250 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
251                                          ptrdiff_t stride)              \
252 {                                                                       \
253     uint64_t half[8 + 9];                                               \
254     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
255     uint8_t * const halfHV = ((uint8_t*)half);                          \
256     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
257                                                    stride, 9);          \
258     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
259     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
260                                         stride, 8, 8);                  \
261 }                                                                       \
262                                                                         \
263 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
264                                          ptrdiff_t stride)              \
265 {                                                                       \
266     uint64_t half[8 + 9];                                               \
267     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
268     uint8_t * const halfHV = ((uint8_t*)half);                          \
269     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
270                                                    stride, 9);          \
271     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
272     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
273                                         stride, 8, 8);                  \
274 }                                                                       \
275                                                                         \
276 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
277                                          ptrdiff_t stride)              \
278 {                                                                       \
279     uint64_t half[8 + 9];                                               \
280     uint8_t * const halfH = ((uint8_t*)half);                           \
281     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
282                                                    stride, 9);          \
283     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
284                                         8, stride, 9);                  \
285     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
286                                                    stride, 8);          \
287 }                                                                       \
288                                                                         \
289 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
290                                          ptrdiff_t stride)              \
291 {                                                                       \
292     uint64_t half[8 + 9];                                               \
293     uint8_t * const halfH = ((uint8_t*)half);                           \
294     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
295                                                    stride, 9);          \
296     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
297                                         stride, 9);                     \
298     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
299                                                    stride, 8);          \
300 }                                                                       \
301                                                                         \
302 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
303                                          ptrdiff_t stride)              \
304 {                                                                       \
305     uint64_t half[9];                                                   \
306     uint8_t * const halfH = ((uint8_t*)half);                           \
307     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
308                                                    stride, 9);          \
309     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
310                                                    stride, 8);          \
311 }                                                                       \
312                                                                         \
313 static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \
314                                            ptrdiff_t stride)            \
315 {                                                                       \
316     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
317 }                                                                       \
318                                                                         \
319 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
320                                           ptrdiff_t stride)             \
321 {                                                                       \
322     uint64_t temp[32];                                                  \
323     uint8_t * const half = (uint8_t*)temp;                              \
324     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
325                                                     stride, 16);        \
326     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
327                                          stride, 16);                   \
328 }                                                                       \
329                                                                         \
330 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
331                                           ptrdiff_t stride)             \
332 {                                                                       \
333     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
334                                                     stride, stride, 16);\
335 }                                                                       \
336                                                                         \
337 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
338                                           ptrdiff_t stride)             \
339 {                                                                       \
340     uint64_t temp[32];                                                  \
341     uint8_t * const half = (uint8_t*)temp;                              \
342     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
343                                                     stride, 16);        \
344     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
345                                          stride, stride, 16);           \
346 }                                                                       \
347                                                                         \
348 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
349                                           ptrdiff_t stride)             \
350 {                                                                       \
351     uint64_t temp[32];                                                  \
352     uint8_t * const half = (uint8_t*)temp;                              \
353     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
354                                                     stride);            \
355     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
356                                          stride, 16);                   \
357 }                                                                       \
358                                                                         \
359 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
360                                           ptrdiff_t stride)             \
361 {                                                                       \
362     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
363                                                     stride, stride);    \
364 }                                                                       \
365                                                                         \
366 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
367                                           ptrdiff_t stride)             \
368 {                                                                       \
369     uint64_t temp[32];                                                  \
370     uint8_t * const half = (uint8_t*)temp;                              \
371     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
372                                                     stride);            \
373     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
374                                          stride, stride, 16);           \
375 }                                                                       \
376                                                                         \
377 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
378                                           ptrdiff_t stride)             \
379 {                                                                       \
380     uint64_t half[16 * 2 + 17 * 2];                                     \
381     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
382     uint8_t * const halfHV = ((uint8_t*)half);                          \
383     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
384                                                     stride, 17);        \
385     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
386                                          stride, 17);                   \
387     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
388                                                     16, 16);            \
389     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
390                                          stride, 16, 16);               \
391 }                                                                       \
392                                                                         \
393 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
394                                           ptrdiff_t stride)             \
395 {                                                                       \
396     uint64_t half[16 * 2 + 17 * 2];                                     \
397     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
398     uint8_t * const halfHV = ((uint8_t*)half);                          \
399     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
400                                                     stride, 17);        \
401     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
402                                          stride, 17);                   \
403     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
404                                                     16, 16);            \
405     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
406                                          stride, 16, 16);               \
407 }                                                                       \
408                                                                         \
409 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
410                                           ptrdiff_t stride)             \
411 {                                                                       \
412     uint64_t half[16 * 2 + 17 * 2];                                     \
413     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
414     uint8_t * const halfHV = ((uint8_t*)half);                          \
415     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
416                                                     stride, 17);        \
417     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
418                                          stride, 17);                   \
419     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
420                                                     16, 16);            \
421     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
422                                          stride, 16, 16);               \
423 }                                                                       \
424                                                                         \
425 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
426                                           ptrdiff_t stride)             \
427 {                                                                       \
428     uint64_t half[16 * 2 + 17 * 2];                                     \
429     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
430     uint8_t * const halfHV = ((uint8_t*)half);                          \
431     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
432                                                     stride, 17);        \
433     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
434                                          stride, 17);                   \
435     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
436                                                     16, 16);            \
437     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
438                                          stride, 16, 16);               \
439 }                                                                       \
440                                                                         \
441 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
442                                           ptrdiff_t stride)             \
443 {                                                                       \
444     uint64_t half[16 * 2 + 17 * 2];                                     \
445     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
446     uint8_t * const halfHV = ((uint8_t*)half);                          \
447     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
448                                                     stride, 17);        \
449     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
450                                                     16, 16);            \
451     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
452                                          stride, 16, 16);               \
453 }                                                                       \
454                                                                         \
455 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
456                                           ptrdiff_t stride)             \
457 {                                                                       \
458     uint64_t half[16 * 2 + 17 * 2];                                     \
459     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
460     uint8_t * const halfHV = ((uint8_t*)half);                          \
461     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
462                                                     stride, 17);        \
463     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
464                                                     16, 16);            \
465     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
466                                          stride, 16, 16);               \
467 }                                                                       \
468                                                                         \
469 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
470                                           ptrdiff_t stride)             \
471 {                                                                       \
472     uint64_t half[17 * 2];                                              \
473     uint8_t * const halfH = ((uint8_t*)half);                           \
474     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
475                                                     stride, 17);        \
476     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
477                                          stride, 17);                   \
478     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
479                                                     stride, 16);        \
480 }                                                                       \
481                                                                         \
482 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
483                                           ptrdiff_t stride)             \
484 {                                                                       \
485     uint64_t half[17 * 2];                                              \
486     uint8_t * const halfH = ((uint8_t*)half);                           \
487     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
488                                                     stride, 17);        \
489     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
490                                          stride, 17);                   \
491     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
492                                                     stride, 16);        \
493 }                                                                       \
494                                                                         \
495 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
496                                           ptrdiff_t stride)             \
497 {                                                                       \
498     uint64_t half[17 * 2];                                              \
499     uint8_t * const halfH = ((uint8_t*)half);                           \
500     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
501                                                     stride, 17);        \
502     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
503                                                     stride, 16);        \
504 }
505
506 QPEL_OP(put_,        _,        mmxext)
507 QPEL_OP(avg_,        _,        mmxext)
508 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
509 #endif /* HAVE_YASM */
510
511 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
512     do {                                                                     \
513     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
514     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
515     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
516     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
517     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
518     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
519     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
520     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
521     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
522     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
523     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
524     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
525     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
526     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
527     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
528     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
529     } while (0)
530
531 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
532                                      int mm_flags)
533 {
534 #if HAVE_MMX_INLINE
535     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
536
537     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
538     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
539     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
540
541     if (!high_bit_depth) {
542         c->clear_block  = ff_clear_block_mmx;
543         c->clear_blocks = ff_clear_blocks_mmx;
544         c->draw_edges   = ff_draw_edges_mmx;
545
546         switch (avctx->idct_algo) {
547         case FF_IDCT_AUTO:
548         case FF_IDCT_SIMPLEMMX:
549             c->idct_put              = ff_simple_idct_put_mmx;
550             c->idct_add              = ff_simple_idct_add_mmx;
551             c->idct                  = ff_simple_idct_mmx;
552             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
553             break;
554         case FF_IDCT_XVIDMMX:
555             c->idct_put              = ff_idct_xvid_mmx_put;
556             c->idct_add              = ff_idct_xvid_mmx_add;
557             c->idct                  = ff_idct_xvid_mmx;
558             break;
559         }
560     }
561
562     c->gmc = ff_gmc_mmx;
563
564     c->add_bytes = ff_add_bytes_mmx;
565 #endif /* HAVE_MMX_INLINE */
566
567 #if HAVE_MMX_EXTERNAL
568     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
569         c->h263_v_loop_filter = ff_h263_v_loop_filter_mmx;
570         c->h263_h_loop_filter = ff_h263_h_loop_filter_mmx;
571     }
572
573     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
574 #endif /* HAVE_MMX_EXTERNAL */
575 }
576
577 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
578                                         int mm_flags)
579 {
580 #if HAVE_MMXEXT_INLINE
581     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
582
583     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
584         c->idct_put = ff_idct_xvid_mmxext_put;
585         c->idct_add = ff_idct_xvid_mmxext_add;
586         c->idct     = ff_idct_xvid_mmxext;
587     }
588 #endif /* HAVE_MMXEXT_INLINE */
589
590 #if HAVE_MMXEXT_EXTERNAL
591     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
592     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
593
594     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
595     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
596     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
597     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
598
599     /* slower than cmov version on AMD */
600     if (!(mm_flags & AV_CPU_FLAG_3DNOW))
601         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
602
603     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
604     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
605
606     if (avctx->flags & CODEC_FLAG_BITEXACT) {
607         c->apply_window_int16 = ff_apply_window_int16_mmxext;
608     } else {
609         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;
610     }
611 #endif /* HAVE_MMXEXT_EXTERNAL */
612 }
613
614 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
615                                      int mm_flags)
616 {
617 #if HAVE_SSE_INLINE
618     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
619
620     if (!high_bit_depth) {
621         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {
622             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
623             c->clear_block  = ff_clear_block_sse;
624             c->clear_blocks = ff_clear_blocks_sse;
625         }
626     }
627
628     c->vector_clipf = ff_vector_clipf_sse;
629 #endif /* HAVE_SSE_INLINE */
630 }
631
632 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
633                                       int mm_flags)
634 {
635 #if HAVE_SSE2_INLINE
636     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
637
638     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
639         c->idct_put              = ff_idct_xvid_sse2_put;
640         c->idct_add              = ff_idct_xvid_sse2_add;
641         c->idct                  = ff_idct_xvid_sse2;
642         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
643     }
644 #endif /* HAVE_SSE2_INLINE */
645
646 #if HAVE_SSE2_EXTERNAL
647     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
648     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
649     if (mm_flags & AV_CPU_FLAG_ATOM) {
650         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
651     } else {
652         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
653     }
654     if (avctx->flags & CODEC_FLAG_BITEXACT) {
655         c->apply_window_int16 = ff_apply_window_int16_sse2;
656     } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {
657         c->apply_window_int16 = ff_apply_window_int16_round_sse2;
658     }
659     c->bswap_buf = ff_bswap32_buf_sse2;
660 #endif /* HAVE_SSE2_EXTERNAL */
661 }
662
663 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
664                                        int mm_flags)
665 {
666 #if HAVE_SSSE3_EXTERNAL
667     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
668     if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe
669         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
670
671     if (mm_flags & AV_CPU_FLAG_ATOM)
672         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;
673     else
674         c->apply_window_int16 = ff_apply_window_int16_ssse3;
675     if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit
676         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
677     c->bswap_buf = ff_bswap32_buf_ssse3;
678 #endif /* HAVE_SSSE3_EXTERNAL */
679 }
680
681 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
682                                       int mm_flags)
683 {
684 #if HAVE_SSE4_EXTERNAL
685     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
686 #endif /* HAVE_SSE4_EXTERNAL */
687 }
688
689 av_cold void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)
690 {
691     int mm_flags = av_get_cpu_flags();
692
693 #if HAVE_7REGS && HAVE_INLINE_ASM
694     if (mm_flags & AV_CPU_FLAG_CMOV)
695         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_cmov;
696 #endif
697
698     if (mm_flags & AV_CPU_FLAG_MMX)
699         dsputil_init_mmx(c, avctx, mm_flags);
700
701     if (mm_flags & AV_CPU_FLAG_MMXEXT)
702         dsputil_init_mmxext(c, avctx, mm_flags);
703
704     if (mm_flags & AV_CPU_FLAG_SSE)
705         dsputil_init_sse(c, avctx, mm_flags);
706
707     if (mm_flags & AV_CPU_FLAG_SSE2)
708         dsputil_init_sse2(c, avctx, mm_flags);
709
710     if (mm_flags & AV_CPU_FLAG_SSSE3)
711         dsputil_init_ssse3(c, avctx, mm_flags);
712
713     if (mm_flags & AV_CPU_FLAG_SSE4)
714         dsputil_init_sse4(c, avctx, mm_flags);
715
716     if (CONFIG_ENCODERS)
717         ff_dsputilenc_init_mmx(c, avctx);
718 }