x86: dsputil: Move fpel declarations to a separate header
[ffmpeg.git] / libavcodec / x86 / dsputil_init.c
1 /*
2  * This file is part of Libav.
3  *
4  * Libav is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * Libav is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with Libav; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include "config.h"
20 #include "libavutil/attributes.h"
21 #include "libavutil/cpu.h"
22 #include "libavutil/internal.h"
23 #include "libavutil/x86/asm.h"
24 #include "libavutil/x86/cpu.h"
25 #include "libavcodec/avcodec.h"
26 #include "libavcodec/dsputil.h"
27 #include "libavcodec/pixels.h"
28 #include "libavcodec/simple_idct.h"
29 #include "libavcodec/version.h"
30 #include "dsputil_x86.h"
31 #include "fpel.h"
32 #include "idct_xvid.h"
33
34 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
35                               int dstStride, int src1Stride, int h);
36 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
37                                      uint8_t *src2, int dstStride,
38                                      int src1Stride, int h);
39 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
40                               int dstStride, int src1Stride, int h);
41 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
42                                int dstStride, int src1Stride, int h);
43 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
44                                int dstStride, int src1Stride, int h);
45 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
46                                       int dstStride, int src1Stride, int h);
47 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
48                                           int dstStride, int srcStride, int h);
49 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
50                                           int dstStride, int srcStride, int h);
51 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
52                                                  int dstStride, int srcStride,
53                                                  int h);
54 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
55                                          int dstStride, int srcStride, int h);
56 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
57                                          int dstStride, int srcStride, int h);
58 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
59                                                 int dstStride, int srcStride,
60                                                 int h);
61 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
62                                           int dstStride, int srcStride);
63 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
64                                           int dstStride, int srcStride);
65 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
66                                                  int dstStride, int srcStride);
67 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
68                                          int dstStride, int srcStride);
69 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
70                                          int dstStride, int srcStride);
71 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
72                                                 int dstStride, int srcStride);
73 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
74 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
75
76 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
77                                       int order);
78 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
79                                     int order);
80 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
81                                                const int16_t *v3,
82                                                int order, int mul);
83 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
84                                              const int16_t *v3,
85                                              int order, int mul);
86 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
87                                               const int16_t *v3,
88                                               int order, int mul);
89
90 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
91 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
92
93 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
94                                           const uint8_t *diff, int w,
95                                           int *left, int *left_top);
96 int ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
97                                       int w, int left);
98 int ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
99                                      int w, int left);
100
101 void ff_vector_clip_int32_mmx(int32_t *dst, const int32_t *src,
102                               int32_t min, int32_t max, unsigned int len);
103 void ff_vector_clip_int32_sse2(int32_t *dst, const int32_t *src,
104                                int32_t min, int32_t max, unsigned int len);
105 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
106                                    int32_t min, int32_t max, unsigned int len);
107 void ff_vector_clip_int32_sse4(int32_t *dst, const int32_t *src,
108                                int32_t min, int32_t max, unsigned int len);
109
110 #if HAVE_YASM
111
112 CALL_2X_PIXELS(ff_avg_pixels16_mmxext, ff_avg_pixels8_mmxext, 8)
113 CALL_2X_PIXELS(ff_put_pixels16_mmxext, ff_put_pixels8_mmxext, 8)
114
115 #define QPEL_OP(OPNAME, RND, MMX)                                       \
116 static void OPNAME ## qpel8_mc00_ ## MMX(uint8_t *dst, uint8_t *src,    \
117                                          ptrdiff_t stride)              \
118 {                                                                       \
119     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
120 }                                                                       \
121                                                                         \
122 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
123                                          ptrdiff_t stride)              \
124 {                                                                       \
125     uint64_t temp[8];                                                   \
126     uint8_t *const half = (uint8_t *) temp;                             \
127     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
128                                                    stride, 8);          \
129     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
130                                         stride, stride, 8);             \
131 }                                                                       \
132                                                                         \
133 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
134                                          ptrdiff_t stride)              \
135 {                                                                       \
136     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
137                                                    stride, 8);          \
138 }                                                                       \
139                                                                         \
140 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
141                                          ptrdiff_t stride)              \
142 {                                                                       \
143     uint64_t temp[8];                                                   \
144     uint8_t *const half = (uint8_t *) temp;                             \
145     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
146                                                    stride, 8);          \
147     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
148                                         stride, 8);                     \
149 }                                                                       \
150                                                                         \
151 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
152                                          ptrdiff_t stride)              \
153 {                                                                       \
154     uint64_t temp[8];                                                   \
155     uint8_t *const half = (uint8_t *) temp;                             \
156     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
157                                                    8, stride);          \
158     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
159                                         stride, stride, 8);             \
160 }                                                                       \
161                                                                         \
162 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
163                                          ptrdiff_t stride)              \
164 {                                                                       \
165     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
166                                                    stride, stride);     \
167 }                                                                       \
168                                                                         \
169 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
170                                          ptrdiff_t stride)              \
171 {                                                                       \
172     uint64_t temp[8];                                                   \
173     uint8_t *const half = (uint8_t *) temp;                             \
174     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
175                                                    8, stride);          \
176     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
177                                         stride, 8);                     \
178 }                                                                       \
179                                                                         \
180 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
181                                          ptrdiff_t stride)              \
182 {                                                                       \
183     uint64_t half[8 + 9];                                               \
184     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
185     uint8_t *const halfHV = (uint8_t *) half;                           \
186     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
187                                                    stride, 9);          \
188     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
189                                         stride, 9);                     \
190     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
191     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
192                                         stride, 8, 8);                  \
193 }                                                                       \
194                                                                         \
195 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
196                                          ptrdiff_t stride)              \
197 {                                                                       \
198     uint64_t half[8 + 9];                                               \
199     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
200     uint8_t *const halfHV = (uint8_t *) half;                           \
201     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
202                                                    stride, 9);          \
203     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
204                                         stride, 9);                     \
205     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
206     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
207                                         stride, 8, 8);                  \
208 }                                                                       \
209                                                                         \
210 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
211                                          ptrdiff_t stride)              \
212 {                                                                       \
213     uint64_t half[8 + 9];                                               \
214     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
215     uint8_t *const halfHV = (uint8_t *) half;                           \
216     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
217                                                    stride, 9);          \
218     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
219                                         stride, 9);                     \
220     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
221     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
222                                         stride, 8, 8);                  \
223 }                                                                       \
224                                                                         \
225 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
226                                          ptrdiff_t stride)              \
227 {                                                                       \
228     uint64_t half[8 + 9];                                               \
229     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
230     uint8_t *const halfHV = (uint8_t *) half;                           \
231     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
232                                                    stride, 9);          \
233     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
234                                         stride, 9);                     \
235     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
236     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
237                                         stride, 8, 8);                  \
238 }                                                                       \
239                                                                         \
240 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
241                                          ptrdiff_t stride)              \
242 {                                                                       \
243     uint64_t half[8 + 9];                                               \
244     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
245     uint8_t *const halfHV = (uint8_t *) half;                           \
246     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
247                                                    stride, 9);          \
248     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
249     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
250                                         stride, 8, 8);                  \
251 }                                                                       \
252                                                                         \
253 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
254                                          ptrdiff_t stride)              \
255 {                                                                       \
256     uint64_t half[8 + 9];                                               \
257     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
258     uint8_t *const halfHV = (uint8_t *) half;                           \
259     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
260                                                    stride, 9);          \
261     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
262     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
263                                         stride, 8, 8);                  \
264 }                                                                       \
265                                                                         \
266 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
267                                          ptrdiff_t stride)              \
268 {                                                                       \
269     uint64_t half[8 + 9];                                               \
270     uint8_t *const halfH = (uint8_t *) half;                            \
271     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
272                                                    stride, 9);          \
273     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
274                                         8, stride, 9);                  \
275     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
276                                                    stride, 8);          \
277 }                                                                       \
278                                                                         \
279 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
280                                          ptrdiff_t stride)              \
281 {                                                                       \
282     uint64_t half[8 + 9];                                               \
283     uint8_t *const halfH = (uint8_t *) half;                            \
284     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
285                                                    stride, 9);          \
286     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
287                                         stride, 9);                     \
288     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
289                                                    stride, 8);          \
290 }                                                                       \
291                                                                         \
292 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
293                                          ptrdiff_t stride)              \
294 {                                                                       \
295     uint64_t half[9];                                                   \
296     uint8_t *const halfH = (uint8_t *) half;                            \
297     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
298                                                    stride, 9);          \
299     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
300                                                    stride, 8);          \
301 }                                                                       \
302                                                                         \
303 static void OPNAME ## qpel16_mc00_ ## MMX(uint8_t *dst, uint8_t *src,   \
304                                           ptrdiff_t stride)             \
305 {                                                                       \
306     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
307 }                                                                       \
308                                                                         \
309 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
310                                           ptrdiff_t stride)             \
311 {                                                                       \
312     uint64_t temp[32];                                                  \
313     uint8_t *const half = (uint8_t *) temp;                             \
314     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
315                                                     stride, 16);        \
316     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
317                                          stride, 16);                   \
318 }                                                                       \
319                                                                         \
320 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
321                                           ptrdiff_t stride)             \
322 {                                                                       \
323     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
324                                                     stride, stride, 16);\
325 }                                                                       \
326                                                                         \
327 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
328                                           ptrdiff_t stride)             \
329 {                                                                       \
330     uint64_t temp[32];                                                  \
331     uint8_t *const half = (uint8_t*) temp;                              \
332     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
333                                                     stride, 16);        \
334     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
335                                          stride, stride, 16);           \
336 }                                                                       \
337                                                                         \
338 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
339                                           ptrdiff_t stride)             \
340 {                                                                       \
341     uint64_t temp[32];                                                  \
342     uint8_t *const half = (uint8_t *) temp;                             \
343     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
344                                                     stride);            \
345     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
346                                          stride, 16);                   \
347 }                                                                       \
348                                                                         \
349 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
350                                           ptrdiff_t stride)             \
351 {                                                                       \
352     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
353                                                     stride, stride);    \
354 }                                                                       \
355                                                                         \
356 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
357                                           ptrdiff_t stride)             \
358 {                                                                       \
359     uint64_t temp[32];                                                  \
360     uint8_t *const half = (uint8_t *) temp;                             \
361     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
362                                                     stride);            \
363     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
364                                          stride, stride, 16);           \
365 }                                                                       \
366                                                                         \
367 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
368                                           ptrdiff_t stride)             \
369 {                                                                       \
370     uint64_t half[16 * 2 + 17 * 2];                                     \
371     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
372     uint8_t *const halfHV = (uint8_t *) half;                           \
373     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
374                                                     stride, 17);        \
375     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
376                                          stride, 17);                   \
377     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
378                                                     16, 16);            \
379     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
380                                          stride, 16, 16);               \
381 }                                                                       \
382                                                                         \
383 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
384                                           ptrdiff_t stride)             \
385 {                                                                       \
386     uint64_t half[16 * 2 + 17 * 2];                                     \
387     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
388     uint8_t *const halfHV = (uint8_t *) half;                           \
389     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
390                                                     stride, 17);        \
391     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
392                                          stride, 17);                   \
393     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
394                                                     16, 16);            \
395     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
396                                          stride, 16, 16);               \
397 }                                                                       \
398                                                                         \
399 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
400                                           ptrdiff_t stride)             \
401 {                                                                       \
402     uint64_t half[16 * 2 + 17 * 2];                                     \
403     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
404     uint8_t *const halfHV = (uint8_t *) half;                           \
405     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
406                                                     stride, 17);        \
407     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
408                                          stride, 17);                   \
409     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
410                                                     16, 16);            \
411     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
412                                          stride, 16, 16);               \
413 }                                                                       \
414                                                                         \
415 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
416                                           ptrdiff_t stride)             \
417 {                                                                       \
418     uint64_t half[16 * 2 + 17 * 2];                                     \
419     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
420     uint8_t *const halfHV = (uint8_t *) half;                           \
421     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
422                                                     stride, 17);        \
423     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
424                                          stride, 17);                   \
425     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
426                                                     16, 16);            \
427     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
428                                          stride, 16, 16);               \
429 }                                                                       \
430                                                                         \
431 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
432                                           ptrdiff_t stride)             \
433 {                                                                       \
434     uint64_t half[16 * 2 + 17 * 2];                                     \
435     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
436     uint8_t *const halfHV = (uint8_t *) half;                           \
437     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
438                                                     stride, 17);        \
439     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
440                                                     16, 16);            \
441     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
442                                          stride, 16, 16);               \
443 }                                                                       \
444                                                                         \
445 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
446                                           ptrdiff_t stride)             \
447 {                                                                       \
448     uint64_t half[16 * 2 + 17 * 2];                                     \
449     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
450     uint8_t *const halfHV = (uint8_t *) half;                           \
451     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
452                                                     stride, 17);        \
453     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
454                                                     16, 16);            \
455     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
456                                          stride, 16, 16);               \
457 }                                                                       \
458                                                                         \
459 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
460                                           ptrdiff_t stride)             \
461 {                                                                       \
462     uint64_t half[17 * 2];                                              \
463     uint8_t *const halfH = (uint8_t *) half;                            \
464     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
465                                                     stride, 17);        \
466     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
467                                          stride, 17);                   \
468     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
469                                                     stride, 16);        \
470 }                                                                       \
471                                                                         \
472 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
473                                           ptrdiff_t stride)             \
474 {                                                                       \
475     uint64_t half[17 * 2];                                              \
476     uint8_t *const halfH = (uint8_t *) half;                            \
477     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
478                                                     stride, 17);        \
479     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
480                                          stride, 17);                   \
481     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
482                                                     stride, 16);        \
483 }                                                                       \
484                                                                         \
485 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
486                                           ptrdiff_t stride)             \
487 {                                                                       \
488     uint64_t half[17 * 2];                                              \
489     uint8_t *const halfH = (uint8_t *) half;                            \
490     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
491                                                     stride, 17);        \
492     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
493                                                     stride, 16);        \
494 }
495
496 QPEL_OP(put_,        _,        mmxext)
497 QPEL_OP(avg_,        _,        mmxext)
498 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
499
500 #endif /* HAVE_YASM */
501
502 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
503 do {                                                                         \
504     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
505     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
506     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
507     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
508     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
509     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
510     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
511     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
512     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
513     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
514     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
515     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
516     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
517     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
518     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
519     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
520 } while (0)
521
522 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
523                                      int cpu_flags, unsigned high_bit_depth)
524 {
525 #if HAVE_MMX_INLINE
526     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
527     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
528     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
529
530     if (!high_bit_depth) {
531         c->clear_block  = ff_clear_block_mmx;
532         c->clear_blocks = ff_clear_blocks_mmx;
533         c->draw_edges   = ff_draw_edges_mmx;
534
535         switch (avctx->idct_algo) {
536         case FF_IDCT_AUTO:
537         case FF_IDCT_SIMPLEMMX:
538             c->idct_put              = ff_simple_idct_put_mmx;
539             c->idct_add              = ff_simple_idct_add_mmx;
540             c->idct                  = ff_simple_idct_mmx;
541             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
542             break;
543         case FF_IDCT_XVIDMMX:
544             c->idct_put              = ff_idct_xvid_mmx_put;
545             c->idct_add              = ff_idct_xvid_mmx_add;
546             c->idct                  = ff_idct_xvid_mmx;
547             break;
548         }
549     }
550
551     c->gmc = ff_gmc_mmx;
552
553     c->add_bytes = ff_add_bytes_mmx;
554 #endif /* HAVE_MMX_INLINE */
555
556 #if HAVE_MMX_EXTERNAL
557     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
558 #endif /* HAVE_MMX_EXTERNAL */
559 }
560
561 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
562                                         int cpu_flags, unsigned high_bit_depth)
563 {
564 #if HAVE_MMXEXT_INLINE
565     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
566         c->idct_put = ff_idct_xvid_mmxext_put;
567         c->idct_add = ff_idct_xvid_mmxext_add;
568         c->idct     = ff_idct_xvid_mmxext;
569     }
570 #endif /* HAVE_MMXEXT_INLINE */
571
572 #if HAVE_MMXEXT_EXTERNAL
573     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
574     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
575
576     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
577     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
578     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
579     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
580
581     /* slower than cmov version on AMD */
582     if (!(cpu_flags & AV_CPU_FLAG_3DNOW))
583         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
584
585     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
586     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
587 #endif /* HAVE_MMXEXT_EXTERNAL */
588 }
589
590 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
591                                      int cpu_flags, unsigned high_bit_depth)
592 {
593 #if HAVE_SSE_INLINE
594     c->vector_clipf = ff_vector_clipf_sse;
595
596 #if FF_API_XVMC
597 FF_DISABLE_DEPRECATION_WARNINGS
598     /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
599     if (CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)
600         return;
601 FF_ENABLE_DEPRECATION_WARNINGS
602 #endif /* FF_API_XVMC */
603
604     if (!high_bit_depth) {
605         c->clear_block  = ff_clear_block_sse;
606         c->clear_blocks = ff_clear_blocks_sse;
607     }
608 #endif /* HAVE_SSE_INLINE */
609 }
610
611 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
612                                       int cpu_flags, unsigned high_bit_depth)
613 {
614 #if HAVE_SSE2_INLINE
615     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
616         c->idct_put              = ff_idct_xvid_sse2_put;
617         c->idct_add              = ff_idct_xvid_sse2_add;
618         c->idct                  = ff_idct_xvid_sse2;
619         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
620     }
621 #endif /* HAVE_SSE2_INLINE */
622
623 #if HAVE_SSE2_EXTERNAL
624     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
625     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
626     if (cpu_flags & AV_CPU_FLAG_ATOM) {
627         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
628     } else {
629         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
630     }
631     c->bswap_buf = ff_bswap32_buf_sse2;
632 #endif /* HAVE_SSE2_EXTERNAL */
633 }
634
635 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
636                                        int cpu_flags, unsigned high_bit_depth)
637 {
638 #if HAVE_SSSE3_EXTERNAL
639     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
640     if (cpu_flags & AV_CPU_FLAG_SSE4) // not really SSE4, just slow on Conroe
641         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
642
643     if (!(cpu_flags & (AV_CPU_FLAG_SSE42 | AV_CPU_FLAG_3DNOW))) // cachesplit
644         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
645     c->bswap_buf = ff_bswap32_buf_ssse3;
646 #endif /* HAVE_SSSE3_EXTERNAL */
647 }
648
649 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
650                                       int cpu_flags, unsigned high_bit_depth)
651 {
652 #if HAVE_SSE4_EXTERNAL
653     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
654 #endif /* HAVE_SSE4_EXTERNAL */
655 }
656
657 av_cold void ff_dsputil_init_x86(DSPContext *c, AVCodecContext *avctx,
658                                  unsigned high_bit_depth)
659 {
660     int cpu_flags = av_get_cpu_flags();
661
662 #if HAVE_7REGS && HAVE_INLINE_ASM
663     if (cpu_flags & AV_CPU_FLAG_CMOV)
664         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_cmov;
665 #endif
666
667     if (X86_MMX(cpu_flags))
668         dsputil_init_mmx(c, avctx, cpu_flags, high_bit_depth);
669
670     if (X86_MMXEXT(cpu_flags))
671         dsputil_init_mmxext(c, avctx, cpu_flags, high_bit_depth);
672
673     if (X86_SSE(cpu_flags))
674         dsputil_init_sse(c, avctx, cpu_flags, high_bit_depth);
675
676     if (X86_SSE2(cpu_flags))
677         dsputil_init_sse2(c, avctx, cpu_flags, high_bit_depth);
678
679     if (EXTERNAL_SSSE3(cpu_flags))
680         dsputil_init_ssse3(c, avctx, cpu_flags, high_bit_depth);
681
682     if (EXTERNAL_SSE4(cpu_flags))
683         dsputil_init_sse4(c, avctx, cpu_flags, high_bit_depth);
684
685     if (CONFIG_ENCODERS)
686         ff_dsputilenc_init_mmx(c, avctx, high_bit_depth);
687 }