Deprecate obsolete XvMC hardware decoding support
[ffmpeg.git] / libavcodec / x86 / dsputil_init.c
1 /*
2  * This file is part of Libav.
3  *
4  * Libav is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * Libav is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with Libav; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include "config.h"
20 #include "libavutil/attributes.h"
21 #include "libavutil/cpu.h"
22 #include "libavutil/x86/asm.h"
23 #include "libavutil/x86/cpu.h"
24 #include "libavcodec/dsputil.h"
25 #include "libavcodec/simple_idct.h"
26 #include "dsputil_x86.h"
27 #include "idct_xvid.h"
28
29 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
30                               int dstStride, int src1Stride, int h);
31 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
32                                      uint8_t *src2, int dstStride,
33                                      int src1Stride, int h);
34 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
35                               int dstStride, int src1Stride, int h);
36 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
37                                int dstStride, int src1Stride, int h);
38 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
39                                int dstStride, int src1Stride, int h);
40 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
41                                       int dstStride, int src1Stride, int h);
42 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
43                                          int dstStride, int srcStride, int h);
44 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
45                                          int dstStride, int srcStride, int h);
46 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
47                                                  int dstStride, int srcStride,
48                                                  int h);
49 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
50                                         int dstStride, int srcStride, int h);
51 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
52                                         int dstStride, int srcStride, int h);
53 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
54                                                 int dstStride, int srcStride,
55                                                 int h);
56 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
57                                          int dstStride, int srcStride);
58 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
59                                          int dstStride, int srcStride);
60 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
61                                                  int dstStride, int srcStride);
62 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
63                                         int dstStride, int srcStride);
64 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
65                                         int dstStride, int srcStride);
66 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
67                                                 int dstStride, int srcStride);
68 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
69 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
70
71 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
72                                       int order);
73 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
74                                     int order);
75 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
76                                                const int16_t *v3,
77                                                int order, int mul);
78 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
79                                              const int16_t *v3,
80                                              int order, int mul);
81 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
82                                               const int16_t *v3,
83                                               int order, int mul);
84
85 void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,
86                                         const int16_t *window, unsigned int len);
87 void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,
88                                       const int16_t *window, unsigned int len);
89 void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,
90                                   const int16_t *window, unsigned int len);
91 void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,
92                                 const int16_t *window, unsigned int len);
93 void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,
94                                  const int16_t *window, unsigned int len);
95 void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,
96                                       const int16_t *window, unsigned int len);
97
98 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
99 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
100
101 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
102                                           const uint8_t *diff, int w,
103                                           int *left, int *left_top);
104 int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
105                                        int w, int left);
106 int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
107                                       int w, int left);
108
109 void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,
110                                    int32_t min, int32_t max, unsigned int len);
111 void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,
112                                    int32_t min, int32_t max, unsigned int len);
113 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
114                                    int32_t min, int32_t max, unsigned int len);
115 void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,
116                                    int32_t min, int32_t max, unsigned int len);
117
118 #if HAVE_YASM
119
120 PIXELS16(static, ff_avg, , , _mmxext)
121 PIXELS16(static, ff_put, , , _mmxext)
122
123 #define QPEL_OP(OPNAME, RND, MMX)                                       \
124 static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \
125                                           ptrdiff_t stride)             \
126 {                                                                       \
127     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
128 }                                                                       \
129                                                                         \
130 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
131                                          ptrdiff_t stride)              \
132 {                                                                       \
133     uint64_t temp[8];                                                   \
134     uint8_t * const half = (uint8_t*)temp;                              \
135     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
136                                                    stride, 8);          \
137     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
138                                         stride, stride, 8);             \
139 }                                                                       \
140                                                                         \
141 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
142                                          ptrdiff_t stride)              \
143 {                                                                       \
144     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
145                                                    stride, 8);          \
146 }                                                                       \
147                                                                         \
148 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
149                                          ptrdiff_t stride)              \
150 {                                                                       \
151     uint64_t temp[8];                                                   \
152     uint8_t * const half = (uint8_t*)temp;                              \
153     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
154                                                    stride, 8);          \
155     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
156                                         stride, 8);                     \
157 }                                                                       \
158                                                                         \
159 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
160                                          ptrdiff_t stride)              \
161 {                                                                       \
162     uint64_t temp[8];                                                   \
163     uint8_t * const half = (uint8_t*)temp;                              \
164     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
165                                                    8, stride);          \
166     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
167                                         stride, stride, 8);             \
168 }                                                                       \
169                                                                         \
170 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
171                                          ptrdiff_t stride)              \
172 {                                                                       \
173     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
174                                                    stride, stride);     \
175 }                                                                       \
176                                                                         \
177 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
178                                          ptrdiff_t stride)              \
179 {                                                                       \
180     uint64_t temp[8];                                                   \
181     uint8_t * const half = (uint8_t*)temp;                              \
182     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
183                                                    8, stride);          \
184     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
185                                         stride, 8);                     \
186 }                                                                       \
187                                                                         \
188 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
189                                          ptrdiff_t stride)              \
190 {                                                                       \
191     uint64_t half[8 + 9];                                               \
192     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
193     uint8_t * const halfHV = ((uint8_t*)half);                          \
194     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
195                                                    stride, 9);          \
196     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
197                                         stride, 9);                     \
198     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
199     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
200                                         stride, 8, 8);                  \
201 }                                                                       \
202                                                                         \
203 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
204                                          ptrdiff_t stride)              \
205 {                                                                       \
206     uint64_t half[8 + 9];                                               \
207     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
208     uint8_t * const halfHV = ((uint8_t*)half);                          \
209     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
210                                                    stride, 9);          \
211     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
212                                         stride, 9);                     \
213     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
214     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
215                                         stride, 8, 8);                  \
216 }                                                                       \
217                                                                         \
218 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
219                                          ptrdiff_t stride)              \
220 {                                                                       \
221     uint64_t half[8 + 9];                                               \
222     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
223     uint8_t * const halfHV = ((uint8_t*)half);                          \
224     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
225                                                    stride, 9);          \
226     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
227                                         stride, 9);                     \
228     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
229     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
230                                         stride, 8, 8);                  \
231 }                                                                       \
232                                                                         \
233 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
234                                          ptrdiff_t stride)              \
235 {                                                                       \
236     uint64_t half[8 + 9];                                               \
237     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
238     uint8_t * const halfHV = ((uint8_t*)half);                          \
239     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
240                                                    stride, 9);          \
241     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
242                                         stride, 9);                     \
243     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
244     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
245                                         stride, 8, 8);                  \
246 }                                                                       \
247                                                                         \
248 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
249                                          ptrdiff_t stride)              \
250 {                                                                       \
251     uint64_t half[8 + 9];                                               \
252     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
253     uint8_t * const halfHV = ((uint8_t*)half);                          \
254     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
255                                                    stride, 9);          \
256     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
257     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
258                                         stride, 8, 8);                  \
259 }                                                                       \
260                                                                         \
261 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
262                                          ptrdiff_t stride)              \
263 {                                                                       \
264     uint64_t half[8 + 9];                                               \
265     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
266     uint8_t * const halfHV = ((uint8_t*)half);                          \
267     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
268                                                    stride, 9);          \
269     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
270     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
271                                         stride, 8, 8);                  \
272 }                                                                       \
273                                                                         \
274 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
275                                          ptrdiff_t stride)              \
276 {                                                                       \
277     uint64_t half[8 + 9];                                               \
278     uint8_t * const halfH = ((uint8_t*)half);                           \
279     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
280                                                    stride, 9);          \
281     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
282                                         8, stride, 9);                  \
283     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
284                                                    stride, 8);          \
285 }                                                                       \
286                                                                         \
287 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
288                                          ptrdiff_t stride)              \
289 {                                                                       \
290     uint64_t half[8 + 9];                                               \
291     uint8_t * const halfH = ((uint8_t*)half);                           \
292     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
293                                                    stride, 9);          \
294     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
295                                         stride, 9);                     \
296     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
297                                                    stride, 8);          \
298 }                                                                       \
299                                                                         \
300 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
301                                          ptrdiff_t stride)              \
302 {                                                                       \
303     uint64_t half[9];                                                   \
304     uint8_t * const halfH = ((uint8_t*)half);                           \
305     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
306                                                    stride, 9);          \
307     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
308                                                    stride, 8);          \
309 }                                                                       \
310                                                                         \
311 static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \
312                                            ptrdiff_t stride)            \
313 {                                                                       \
314     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
315 }                                                                       \
316                                                                         \
317 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
318                                           ptrdiff_t stride)             \
319 {                                                                       \
320     uint64_t temp[32];                                                  \
321     uint8_t * const half = (uint8_t*)temp;                              \
322     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
323                                                     stride, 16);        \
324     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
325                                          stride, 16);                   \
326 }                                                                       \
327                                                                         \
328 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
329                                           ptrdiff_t stride)             \
330 {                                                                       \
331     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
332                                                     stride, stride, 16);\
333 }                                                                       \
334                                                                         \
335 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
336                                           ptrdiff_t stride)             \
337 {                                                                       \
338     uint64_t temp[32];                                                  \
339     uint8_t * const half = (uint8_t*)temp;                              \
340     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
341                                                     stride, 16);        \
342     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
343                                          stride, stride, 16);           \
344 }                                                                       \
345                                                                         \
346 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
347                                           ptrdiff_t stride)             \
348 {                                                                       \
349     uint64_t temp[32];                                                  \
350     uint8_t * const half = (uint8_t*)temp;                              \
351     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
352                                                     stride);            \
353     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
354                                          stride, 16);                   \
355 }                                                                       \
356                                                                         \
357 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
358                                           ptrdiff_t stride)             \
359 {                                                                       \
360     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
361                                                     stride, stride);    \
362 }                                                                       \
363                                                                         \
364 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
365                                           ptrdiff_t stride)             \
366 {                                                                       \
367     uint64_t temp[32];                                                  \
368     uint8_t * const half = (uint8_t*)temp;                              \
369     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
370                                                     stride);            \
371     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
372                                          stride, stride, 16);           \
373 }                                                                       \
374                                                                         \
375 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
376                                           ptrdiff_t stride)             \
377 {                                                                       \
378     uint64_t half[16 * 2 + 17 * 2];                                     \
379     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
380     uint8_t * const halfHV = ((uint8_t*)half);                          \
381     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
382                                                     stride, 17);        \
383     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
384                                          stride, 17);                   \
385     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
386                                                     16, 16);            \
387     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
388                                          stride, 16, 16);               \
389 }                                                                       \
390                                                                         \
391 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
392                                           ptrdiff_t stride)             \
393 {                                                                       \
394     uint64_t half[16 * 2 + 17 * 2];                                     \
395     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
396     uint8_t * const halfHV = ((uint8_t*)half);                          \
397     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
398                                                     stride, 17);        \
399     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
400                                          stride, 17);                   \
401     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
402                                                     16, 16);            \
403     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
404                                          stride, 16, 16);               \
405 }                                                                       \
406                                                                         \
407 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
408                                           ptrdiff_t stride)             \
409 {                                                                       \
410     uint64_t half[16 * 2 + 17 * 2];                                     \
411     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
412     uint8_t * const halfHV = ((uint8_t*)half);                          \
413     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
414                                                     stride, 17);        \
415     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
416                                          stride, 17);                   \
417     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
418                                                     16, 16);            \
419     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
420                                          stride, 16, 16);               \
421 }                                                                       \
422                                                                         \
423 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
424                                           ptrdiff_t stride)             \
425 {                                                                       \
426     uint64_t half[16 * 2 + 17 * 2];                                     \
427     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
428     uint8_t * const halfHV = ((uint8_t*)half);                          \
429     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
430                                                     stride, 17);        \
431     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
432                                          stride, 17);                   \
433     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
434                                                     16, 16);            \
435     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
436                                          stride, 16, 16);               \
437 }                                                                       \
438                                                                         \
439 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
440                                           ptrdiff_t stride)             \
441 {                                                                       \
442     uint64_t half[16 * 2 + 17 * 2];                                     \
443     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
444     uint8_t * const halfHV = ((uint8_t*)half);                          \
445     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
446                                                     stride, 17);        \
447     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
448                                                     16, 16);            \
449     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
450                                          stride, 16, 16);               \
451 }                                                                       \
452                                                                         \
453 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
454                                           ptrdiff_t stride)             \
455 {                                                                       \
456     uint64_t half[16 * 2 + 17 * 2];                                     \
457     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
458     uint8_t * const halfHV = ((uint8_t*)half);                          \
459     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
460                                                     stride, 17);        \
461     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
462                                                     16, 16);            \
463     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
464                                          stride, 16, 16);               \
465 }                                                                       \
466                                                                         \
467 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
468                                           ptrdiff_t stride)             \
469 {                                                                       \
470     uint64_t half[17 * 2];                                              \
471     uint8_t * const halfH = ((uint8_t*)half);                           \
472     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
473                                                     stride, 17);        \
474     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
475                                          stride, 17);                   \
476     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
477                                                     stride, 16);        \
478 }                                                                       \
479                                                                         \
480 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
481                                           ptrdiff_t stride)             \
482 {                                                                       \
483     uint64_t half[17 * 2];                                              \
484     uint8_t * const halfH = ((uint8_t*)half);                           \
485     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
486                                                     stride, 17);        \
487     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
488                                          stride, 17);                   \
489     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
490                                                     stride, 16);        \
491 }                                                                       \
492                                                                         \
493 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
494                                           ptrdiff_t stride)             \
495 {                                                                       \
496     uint64_t half[17 * 2];                                              \
497     uint8_t * const halfH = ((uint8_t*)half);                           \
498     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
499                                                     stride, 17);        \
500     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
501                                                     stride, 16);        \
502 }
503
504 QPEL_OP(put_,        _,        mmxext)
505 QPEL_OP(avg_,        _,        mmxext)
506 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
507 #endif /* HAVE_YASM */
508
509 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
510     do {                                                                     \
511     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
512     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
513     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
514     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
515     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
516     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
517     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
518     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
519     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
520     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
521     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
522     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
523     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
524     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
525     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
526     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
527     } while (0)
528
529 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
530                                      int cpu_flags)
531 {
532 #if HAVE_MMX_INLINE
533     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
534
535     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
536     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
537     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
538
539     if (!high_bit_depth) {
540         c->clear_block  = ff_clear_block_mmx;
541         c->clear_blocks = ff_clear_blocks_mmx;
542         c->draw_edges   = ff_draw_edges_mmx;
543
544         switch (avctx->idct_algo) {
545         case FF_IDCT_AUTO:
546         case FF_IDCT_SIMPLEMMX:
547             c->idct_put              = ff_simple_idct_put_mmx;
548             c->idct_add              = ff_simple_idct_add_mmx;
549             c->idct                  = ff_simple_idct_mmx;
550             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
551             break;
552         case FF_IDCT_XVIDMMX:
553             c->idct_put              = ff_idct_xvid_mmx_put;
554             c->idct_add              = ff_idct_xvid_mmx_add;
555             c->idct                  = ff_idct_xvid_mmx;
556             break;
557         }
558     }
559
560     c->gmc = ff_gmc_mmx;
561
562     c->add_bytes = ff_add_bytes_mmx;
563 #endif /* HAVE_MMX_INLINE */
564
565 #if HAVE_MMX_EXTERNAL
566     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
567 #endif /* HAVE_MMX_EXTERNAL */
568 }
569
570 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
571                                         int cpu_flags)
572 {
573 #if HAVE_MMXEXT_INLINE
574     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
575
576     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
577         c->idct_put = ff_idct_xvid_mmxext_put;
578         c->idct_add = ff_idct_xvid_mmxext_add;
579         c->idct     = ff_idct_xvid_mmxext;
580     }
581 #endif /* HAVE_MMXEXT_INLINE */
582
583 #if HAVE_MMXEXT_EXTERNAL
584     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
585     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
586
587     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
588     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
589     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
590     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
591
592     /* slower than cmov version on AMD */
593     if (!(cpu_flags & AV_CPU_FLAG_3DNOW))
594         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
595
596     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
597     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
598
599     if (avctx->flags & CODEC_FLAG_BITEXACT) {
600         c->apply_window_int16 = ff_apply_window_int16_mmxext;
601     } else {
602         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;
603     }
604 #endif /* HAVE_MMXEXT_EXTERNAL */
605 }
606
607 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
608                                      int cpu_flags)
609 {
610 #if HAVE_SSE_INLINE
611     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
612
613     if (!high_bit_depth) {
614 #if FF_API_XVMC
615         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {
616             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
617 #endif /* FF_API_XVMC */
618         c->clear_block  = ff_clear_block_sse;
619         c->clear_blocks = ff_clear_blocks_sse;
620 #if FF_API_XVMC
621         }
622 #endif /* FF_API_XVMC */
623     }
624
625     c->vector_clipf = ff_vector_clipf_sse;
626 #endif /* HAVE_SSE_INLINE */
627 }
628
629 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
630                                       int cpu_flags)
631 {
632 #if HAVE_SSE2_INLINE
633     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
634
635     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
636         c->idct_put              = ff_idct_xvid_sse2_put;
637         c->idct_add              = ff_idct_xvid_sse2_add;
638         c->idct                  = ff_idct_xvid_sse2;
639         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
640     }
641 #endif /* HAVE_SSE2_INLINE */
642
643 #if HAVE_SSE2_EXTERNAL
644     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
645     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
646     if (cpu_flags & AV_CPU_FLAG_ATOM) {
647         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
648     } else {
649         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
650     }
651     if (avctx->flags & CODEC_FLAG_BITEXACT) {
652         c->apply_window_int16 = ff_apply_window_int16_sse2;
653     } else if (!(cpu_flags & AV_CPU_FLAG_SSE2SLOW)) {
654         c->apply_window_int16 = ff_apply_window_int16_round_sse2;
655     }
656     c->bswap_buf = ff_bswap32_buf_sse2;
657 #endif /* HAVE_SSE2_EXTERNAL */
658 }
659
660 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
661                                        int cpu_flags)
662 {
663 #if HAVE_SSSE3_EXTERNAL
664     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
665     if (cpu_flags & AV_CPU_FLAG_SSE4) // not really SSE4, just slow on Conroe
666         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
667
668     if (cpu_flags & AV_CPU_FLAG_ATOM)
669         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;
670     else
671         c->apply_window_int16 = ff_apply_window_int16_ssse3;
672     if (!(cpu_flags & (AV_CPU_FLAG_SSE42 | AV_CPU_FLAG_3DNOW))) // cachesplit
673         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
674     c->bswap_buf = ff_bswap32_buf_ssse3;
675 #endif /* HAVE_SSSE3_EXTERNAL */
676 }
677
678 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
679                                       int cpu_flags)
680 {
681 #if HAVE_SSE4_EXTERNAL
682     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
683 #endif /* HAVE_SSE4_EXTERNAL */
684 }
685
686 av_cold void ff_dsputil_init_x86(DSPContext *c, AVCodecContext *avctx)
687 {
688     int cpu_flags = av_get_cpu_flags();
689
690 #if HAVE_7REGS && HAVE_INLINE_ASM
691     if (cpu_flags & AV_CPU_FLAG_CMOV)
692         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_cmov;
693 #endif
694
695     if (X86_MMX(cpu_flags))
696         dsputil_init_mmx(c, avctx, cpu_flags);
697
698     if (X86_MMXEXT(cpu_flags))
699         dsputil_init_mmxext(c, avctx, cpu_flags);
700
701     if (X86_SSE(cpu_flags))
702         dsputil_init_sse(c, avctx, cpu_flags);
703
704     if (X86_SSE2(cpu_flags))
705         dsputil_init_sse2(c, avctx, cpu_flags);
706
707     if (EXTERNAL_SSSE3(cpu_flags))
708         dsputil_init_ssse3(c, avctx, cpu_flags);
709
710     if (EXTERNAL_SSE4(cpu_flags))
711         dsputil_init_sse4(c, avctx, cpu_flags);
712
713     if (CONFIG_ENCODERS)
714         ff_dsputilenc_init_mmx(c, avctx);
715 }