Merge commit '245b76a108585b6fb52eebc2626c472d6fa530dc'
[ffmpeg.git] / libavcodec / x86 / dsputil_init.c
1 /*
2  * Copyright (c) 2000, 2001 Fabrice Bellard
3  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 #include "config.h"
23 #include "libavutil/attributes.h"
24 #include "libavutil/cpu.h"
25 #include "libavutil/x86/asm.h"
26 #include "libavcodec/dsputil.h"
27 #include "libavcodec/simple_idct.h"
28 #include "dsputil_mmx.h"
29 #include "idct_xvid.h"
30
31 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
32                               int dstStride, int src1Stride, int h);
33 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
34                                      uint8_t *src2, int dstStride,
35                                      int src1Stride, int h);
36 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
37                               int dstStride, int src1Stride, int h);
38 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
39                                int dstStride, int src1Stride, int h);
40 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
41                                int dstStride, int src1Stride, int h);
42 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
43                                       int dstStride, int src1Stride, int h);
44 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
45                                          int dstStride, int srcStride, int h);
46 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
47                                          int dstStride, int srcStride, int h);
48 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
49                                                  int dstStride, int srcStride,
50                                                  int h);
51 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
52                                         int dstStride, int srcStride, int h);
53 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
54                                         int dstStride, int srcStride, int h);
55 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
56                                                 int dstStride, int srcStride,
57                                                 int h);
58 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
59                                          int dstStride, int srcStride);
60 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
61                                          int dstStride, int srcStride);
62 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
63                                                  int dstStride, int srcStride);
64 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
65                                         int dstStride, int srcStride);
66 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
67                                         int dstStride, int srcStride);
68 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
69                                                 int dstStride, int srcStride);
70 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
71 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
72
73 void ff_h263_v_loop_filter_mmx(uint8_t *src, int stride, int qscale);
74 void ff_h263_h_loop_filter_mmx(uint8_t *src, int stride, int qscale);
75
76 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
77                                       int order);
78 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
79                                     int order);
80 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
81                                                const int16_t *v3,
82                                                int order, int mul);
83 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
84                                              const int16_t *v3,
85                                              int order, int mul);
86 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
87                                               const int16_t *v3,
88                                               int order, int mul);
89
90 void ff_apply_window_int16_round_mmxext(int16_t *output, const int16_t *input,
91                                         const int16_t *window, unsigned int len);
92 void ff_apply_window_int16_round_sse2(int16_t *output, const int16_t *input,
93                                       const int16_t *window, unsigned int len);
94 void ff_apply_window_int16_mmxext(int16_t *output, const int16_t *input,
95                                   const int16_t *window, unsigned int len);
96 void ff_apply_window_int16_sse2(int16_t *output, const int16_t *input,
97                                 const int16_t *window, unsigned int len);
98 void ff_apply_window_int16_ssse3(int16_t *output, const int16_t *input,
99                                  const int16_t *window, unsigned int len);
100 void ff_apply_window_int16_ssse3_atom(int16_t *output, const int16_t *input,
101                                       const int16_t *window, unsigned int len);
102
103 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
104 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
105
106 void ff_add_hfyu_median_prediction_mmxext(uint8_t *dst, const uint8_t *top,
107                                           const uint8_t *diff, int w,
108                                           int *left, int *left_top);
109 int  ff_add_hfyu_left_prediction_ssse3(uint8_t *dst, const uint8_t *src,
110                                        int w, int left);
111 int  ff_add_hfyu_left_prediction_sse4(uint8_t *dst, const uint8_t *src,
112                                       int w, int left);
113
114 void ff_vector_clip_int32_mmx     (int32_t *dst, const int32_t *src,
115                                    int32_t min, int32_t max, unsigned int len);
116 void ff_vector_clip_int32_sse2    (int32_t *dst, const int32_t *src,
117                                    int32_t min, int32_t max, unsigned int len);
118 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
119                                    int32_t min, int32_t max, unsigned int len);
120 void ff_vector_clip_int32_sse4    (int32_t *dst, const int32_t *src,
121                                    int32_t min, int32_t max, unsigned int len);
122
123 #if HAVE_YASM
124
125 PIXELS16(static, ff_avg, , , _mmxext)
126 PIXELS16(static, ff_put, , , _mmxext)
127
128 #define QPEL_OP(OPNAME, RND, MMX)                                       \
129 static void OPNAME ## qpel8_mc00_ ## MMX (uint8_t *dst, uint8_t *src,   \
130                                           ptrdiff_t stride)             \
131 {                                                                       \
132     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
133 }                                                                       \
134                                                                         \
135 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
136                                          ptrdiff_t stride)              \
137 {                                                                       \
138     uint64_t temp[8];                                                   \
139     uint8_t * const half = (uint8_t*)temp;                              \
140     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
141                                                    stride, 8);          \
142     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
143                                         stride, stride, 8);             \
144 }                                                                       \
145                                                                         \
146 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
147                                          ptrdiff_t stride)              \
148 {                                                                       \
149     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
150                                                    stride, 8);          \
151 }                                                                       \
152                                                                         \
153 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
154                                          ptrdiff_t stride)              \
155 {                                                                       \
156     uint64_t temp[8];                                                   \
157     uint8_t * const half = (uint8_t*)temp;                              \
158     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
159                                                    stride, 8);          \
160     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
161                                         stride, 8);                     \
162 }                                                                       \
163                                                                         \
164 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
165                                          ptrdiff_t stride)              \
166 {                                                                       \
167     uint64_t temp[8];                                                   \
168     uint8_t * const half = (uint8_t*)temp;                              \
169     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
170                                                    8, stride);          \
171     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
172                                         stride, stride, 8);             \
173 }                                                                       \
174                                                                         \
175 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
176                                          ptrdiff_t stride)              \
177 {                                                                       \
178     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
179                                                    stride, stride);     \
180 }                                                                       \
181                                                                         \
182 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
183                                          ptrdiff_t stride)              \
184 {                                                                       \
185     uint64_t temp[8];                                                   \
186     uint8_t * const half = (uint8_t*)temp;                              \
187     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
188                                                    8, stride);          \
189     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
190                                         stride, 8);                     \
191 }                                                                       \
192                                                                         \
193 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
194                                          ptrdiff_t stride)              \
195 {                                                                       \
196     uint64_t half[8 + 9];                                               \
197     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
198     uint8_t * const halfHV = ((uint8_t*)half);                          \
199     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
200                                                    stride, 9);          \
201     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
202                                         stride, 9);                     \
203     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
204     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
205                                         stride, 8, 8);                  \
206 }                                                                       \
207                                                                         \
208 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
209                                          ptrdiff_t stride)              \
210 {                                                                       \
211     uint64_t half[8 + 9];                                               \
212     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
213     uint8_t * const halfHV = ((uint8_t*)half);                          \
214     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
215                                                    stride, 9);          \
216     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
217                                         stride, 9);                     \
218     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
219     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
220                                         stride, 8, 8);                  \
221 }                                                                       \
222                                                                         \
223 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
224                                          ptrdiff_t stride)              \
225 {                                                                       \
226     uint64_t half[8 + 9];                                               \
227     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
228     uint8_t * const halfHV = ((uint8_t*)half);                          \
229     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
230                                                    stride, 9);          \
231     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
232                                         stride, 9);                     \
233     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
234     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
235                                         stride, 8, 8);                  \
236 }                                                                       \
237                                                                         \
238 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
239                                          ptrdiff_t stride)              \
240 {                                                                       \
241     uint64_t half[8 + 9];                                               \
242     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
243     uint8_t * const halfHV = ((uint8_t*)half);                          \
244     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
245                                                    stride, 9);          \
246     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
247                                         stride, 9);                     \
248     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
249     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
250                                         stride, 8, 8);                  \
251 }                                                                       \
252                                                                         \
253 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
254                                          ptrdiff_t stride)              \
255 {                                                                       \
256     uint64_t half[8 + 9];                                               \
257     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
258     uint8_t * const halfHV = ((uint8_t*)half);                          \
259     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
260                                                    stride, 9);          \
261     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
262     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
263                                         stride, 8, 8);                  \
264 }                                                                       \
265                                                                         \
266 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
267                                          ptrdiff_t stride)              \
268 {                                                                       \
269     uint64_t half[8 + 9];                                               \
270     uint8_t * const halfH  = ((uint8_t*)half) + 64;                     \
271     uint8_t * const halfHV = ((uint8_t*)half);                          \
272     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
273                                                    stride, 9);          \
274     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
275     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
276                                         stride, 8, 8);                  \
277 }                                                                       \
278                                                                         \
279 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
280                                          ptrdiff_t stride)              \
281 {                                                                       \
282     uint64_t half[8 + 9];                                               \
283     uint8_t * const halfH = ((uint8_t*)half);                           \
284     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
285                                                    stride, 9);          \
286     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
287                                         8, stride, 9);                  \
288     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
289                                                    stride, 8);          \
290 }                                                                       \
291                                                                         \
292 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
293                                          ptrdiff_t stride)              \
294 {                                                                       \
295     uint64_t half[8 + 9];                                               \
296     uint8_t * const halfH = ((uint8_t*)half);                           \
297     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
298                                                    stride, 9);          \
299     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
300                                         stride, 9);                     \
301     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
302                                                    stride, 8);          \
303 }                                                                       \
304                                                                         \
305 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
306                                          ptrdiff_t stride)              \
307 {                                                                       \
308     uint64_t half[9];                                                   \
309     uint8_t * const halfH = ((uint8_t*)half);                           \
310     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
311                                                    stride, 9);          \
312     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
313                                                    stride, 8);          \
314 }                                                                       \
315                                                                         \
316 static void OPNAME ## qpel16_mc00_ ## MMX (uint8_t *dst, uint8_t *src,  \
317                                            ptrdiff_t stride)            \
318 {                                                                       \
319     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
320 }                                                                       \
321                                                                         \
322 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
323                                           ptrdiff_t stride)             \
324 {                                                                       \
325     uint64_t temp[32];                                                  \
326     uint8_t * const half = (uint8_t*)temp;                              \
327     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
328                                                     stride, 16);        \
329     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
330                                          stride, 16);                   \
331 }                                                                       \
332                                                                         \
333 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
334                                           ptrdiff_t stride)             \
335 {                                                                       \
336     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
337                                                     stride, stride, 16);\
338 }                                                                       \
339                                                                         \
340 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
341                                           ptrdiff_t stride)             \
342 {                                                                       \
343     uint64_t temp[32];                                                  \
344     uint8_t * const half = (uint8_t*)temp;                              \
345     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
346                                                     stride, 16);        \
347     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
348                                          stride, stride, 16);           \
349 }                                                                       \
350                                                                         \
351 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
352                                           ptrdiff_t stride)             \
353 {                                                                       \
354     uint64_t temp[32];                                                  \
355     uint8_t * const half = (uint8_t*)temp;                              \
356     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
357                                                     stride);            \
358     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
359                                          stride, 16);                   \
360 }                                                                       \
361                                                                         \
362 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
363                                           ptrdiff_t stride)             \
364 {                                                                       \
365     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
366                                                     stride, stride);    \
367 }                                                                       \
368                                                                         \
369 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
370                                           ptrdiff_t stride)             \
371 {                                                                       \
372     uint64_t temp[32];                                                  \
373     uint8_t * const half = (uint8_t*)temp;                              \
374     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
375                                                     stride);            \
376     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
377                                          stride, stride, 16);           \
378 }                                                                       \
379                                                                         \
380 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
381                                           ptrdiff_t stride)             \
382 {                                                                       \
383     uint64_t half[16 * 2 + 17 * 2];                                     \
384     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
385     uint8_t * const halfHV = ((uint8_t*)half);                          \
386     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
387                                                     stride, 17);        \
388     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
389                                          stride, 17);                   \
390     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
391                                                     16, 16);            \
392     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
393                                          stride, 16, 16);               \
394 }                                                                       \
395                                                                         \
396 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
397                                           ptrdiff_t stride)             \
398 {                                                                       \
399     uint64_t half[16 * 2 + 17 * 2];                                     \
400     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
401     uint8_t * const halfHV = ((uint8_t*)half);                          \
402     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
403                                                     stride, 17);        \
404     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
405                                          stride, 17);                   \
406     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
407                                                     16, 16);            \
408     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
409                                          stride, 16, 16);               \
410 }                                                                       \
411                                                                         \
412 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
413                                           ptrdiff_t stride)             \
414 {                                                                       \
415     uint64_t half[16 * 2 + 17 * 2];                                     \
416     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
417     uint8_t * const halfHV = ((uint8_t*)half);                          \
418     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
419                                                     stride, 17);        \
420     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
421                                          stride, 17);                   \
422     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
423                                                     16, 16);            \
424     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
425                                          stride, 16, 16);               \
426 }                                                                       \
427                                                                         \
428 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
429                                           ptrdiff_t stride)             \
430 {                                                                       \
431     uint64_t half[16 * 2 + 17 * 2];                                     \
432     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
433     uint8_t * const halfHV = ((uint8_t*)half);                          \
434     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
435                                                     stride, 17);        \
436     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
437                                          stride, 17);                   \
438     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
439                                                     16, 16);            \
440     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
441                                          stride, 16, 16);               \
442 }                                                                       \
443                                                                         \
444 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
445                                           ptrdiff_t stride)             \
446 {                                                                       \
447     uint64_t half[16 * 2 + 17 * 2];                                     \
448     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
449     uint8_t * const halfHV = ((uint8_t*)half);                          \
450     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
451                                                     stride, 17);        \
452     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
453                                                     16, 16);            \
454     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
455                                          stride, 16, 16);               \
456 }                                                                       \
457                                                                         \
458 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
459                                           ptrdiff_t stride)             \
460 {                                                                       \
461     uint64_t half[16 * 2 + 17 * 2];                                     \
462     uint8_t * const halfH  = ((uint8_t*)half) + 256;                    \
463     uint8_t * const halfHV = ((uint8_t*)half);                          \
464     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
465                                                     stride, 17);        \
466     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
467                                                     16, 16);            \
468     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
469                                          stride, 16, 16);               \
470 }                                                                       \
471                                                                         \
472 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
473                                           ptrdiff_t stride)             \
474 {                                                                       \
475     uint64_t half[17 * 2];                                              \
476     uint8_t * const halfH = ((uint8_t*)half);                           \
477     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
478                                                     stride, 17);        \
479     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
480                                          stride, 17);                   \
481     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
482                                                     stride, 16);        \
483 }                                                                       \
484                                                                         \
485 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
486                                           ptrdiff_t stride)             \
487 {                                                                       \
488     uint64_t half[17 * 2];                                              \
489     uint8_t * const halfH = ((uint8_t*)half);                           \
490     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
491                                                     stride, 17);        \
492     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
493                                          stride, 17);                   \
494     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
495                                                     stride, 16);        \
496 }                                                                       \
497                                                                         \
498 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
499                                           ptrdiff_t stride)             \
500 {                                                                       \
501     uint64_t half[17 * 2];                                              \
502     uint8_t * const halfH = ((uint8_t*)half);                           \
503     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
504                                                     stride, 17);        \
505     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
506                                                     stride, 16);        \
507 }
508
509 QPEL_OP(put_,        _,        mmxext)
510 QPEL_OP(avg_,        _,        mmxext)
511 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
512 #endif /* HAVE_YASM */
513
514 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
515     do {                                                                     \
516     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
517     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
518     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
519     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
520     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
521     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
522     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
523     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
524     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
525     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
526     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
527     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
528     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
529     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
530     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
531     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
532     } while (0)
533
534 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
535                                      int mm_flags)
536 {
537 #if HAVE_MMX_INLINE
538     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
539
540     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
541     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
542     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
543
544     if (!high_bit_depth) {
545         c->clear_block  = ff_clear_block_mmx;
546         c->clear_blocks = ff_clear_blocks_mmx;
547         c->draw_edges   = ff_draw_edges_mmx;
548     }
549
550 #if CONFIG_VIDEODSP && (ARCH_X86_32 || !HAVE_YASM)
551     c->gmc = ff_gmc_mmx;
552 #endif
553
554     c->add_bytes = ff_add_bytes_mmx;
555 #endif /* HAVE_MMX_INLINE */
556
557 #if HAVE_MMX_EXTERNAL
558     if (CONFIG_H263_DECODER || CONFIG_H263_ENCODER) {
559         c->h263_v_loop_filter = ff_h263_v_loop_filter_mmx;
560         c->h263_h_loop_filter = ff_h263_h_loop_filter_mmx;
561     }
562
563     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
564 #endif /* HAVE_MMX_EXTERNAL */
565 }
566
567 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
568                                         int mm_flags)
569 {
570 #if HAVE_MMXEXT_EXTERNAL
571     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
572     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
573
574     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
575     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
576     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
577     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
578
579     /* slower than cmov version on AMD */
580     if (!(mm_flags & AV_CPU_FLAG_3DNOW))
581         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_mmxext;
582
583     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
584     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
585
586     if (avctx->flags & CODEC_FLAG_BITEXACT) {
587         c->apply_window_int16 = ff_apply_window_int16_mmxext;
588     } else {
589         c->apply_window_int16 = ff_apply_window_int16_round_mmxext;
590     }
591 #endif /* HAVE_MMXEXT_EXTERNAL */
592 }
593
594 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
595                                      int mm_flags)
596 {
597 #if HAVE_SSE_INLINE
598     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
599
600     if (!high_bit_depth) {
601         if (!(CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)) {
602             /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
603             c->clear_block  = ff_clear_block_sse;
604             c->clear_blocks = ff_clear_blocks_sse;
605         }
606     }
607
608     c->vector_clipf = ff_vector_clipf_sse;
609 #endif /* HAVE_SSE_INLINE */
610
611 #if HAVE_YASM
612 #if HAVE_INLINE_ASM && CONFIG_VIDEODSP
613     c->gmc = ff_gmc_sse;
614 #endif
615 #endif /* HAVE_YASM */
616 }
617
618 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
619                                       int mm_flags)
620 {
621 #if HAVE_SSE2_INLINE
622     const int high_bit_depth = avctx->bits_per_raw_sample > 8;
623
624     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
625         c->idct_put              = ff_idct_xvid_sse2_put;
626         c->idct_add              = ff_idct_xvid_sse2_add;
627         c->idct                  = ff_idct_xvid_sse2;
628         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
629     }
630 #endif /* HAVE_SSE2_INLINE */
631
632 #if HAVE_SSE2_EXTERNAL
633     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
634     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
635     if (mm_flags & AV_CPU_FLAG_ATOM) {
636         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
637     } else {
638         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
639     }
640     if (avctx->flags & CODEC_FLAG_BITEXACT) {
641         c->apply_window_int16 = ff_apply_window_int16_sse2;
642     } else if (!(mm_flags & AV_CPU_FLAG_SSE2SLOW)) {
643         c->apply_window_int16 = ff_apply_window_int16_round_sse2;
644     }
645     c->bswap_buf = ff_bswap32_buf_sse2;
646 #endif /* HAVE_SSE2_EXTERNAL */
647 }
648
649 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
650                                        int mm_flags)
651 {
652 #if HAVE_SSSE3_EXTERNAL
653     c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_ssse3;
654     if (mm_flags & AV_CPU_FLAG_SSE4) // not really sse4, just slow on Conroe
655         c->add_hfyu_left_prediction = ff_add_hfyu_left_prediction_sse4;
656
657     if (mm_flags & AV_CPU_FLAG_ATOM)
658         c->apply_window_int16 = ff_apply_window_int16_ssse3_atom;
659     else
660         c->apply_window_int16 = ff_apply_window_int16_ssse3;
661     if (!(mm_flags & (AV_CPU_FLAG_SSE42|AV_CPU_FLAG_3DNOW))) // cachesplit
662         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
663     c->bswap_buf = ff_bswap32_buf_ssse3;
664 #endif /* HAVE_SSSE3_EXTERNAL */
665 }
666
667 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
668                                       int mm_flags)
669 {
670 #if HAVE_SSE4_EXTERNAL
671     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
672 #endif /* HAVE_SSE4_EXTERNAL */
673 }
674
675 av_cold void ff_dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx)
676 {
677     int mm_flags = av_get_cpu_flags();
678
679 #if HAVE_7REGS && HAVE_INLINE_ASM
680     if (mm_flags & AV_CPU_FLAG_CMOV)
681         c->add_hfyu_median_prediction = ff_add_hfyu_median_prediction_cmov;
682 #endif
683
684     if (mm_flags & AV_CPU_FLAG_MMX) {
685 #if HAVE_INLINE_ASM
686         const int idct_algo = avctx->idct_algo;
687
688         if (avctx->lowres == 0 && avctx->bits_per_raw_sample <= 8) {
689             if (idct_algo == FF_IDCT_AUTO || idct_algo == FF_IDCT_SIMPLEMMX) {
690                 c->idct_put              = ff_simple_idct_put_mmx;
691                 c->idct_add              = ff_simple_idct_add_mmx;
692                 c->idct                  = ff_simple_idct_mmx;
693                 c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
694             } else if (idct_algo == FF_IDCT_XVIDMMX) {
695                 if (mm_flags & AV_CPU_FLAG_SSE2) {
696                     c->idct_put              = ff_idct_xvid_sse2_put;
697                     c->idct_add              = ff_idct_xvid_sse2_add;
698                     c->idct                  = ff_idct_xvid_sse2;
699                     c->idct_permutation_type = FF_SSE2_IDCT_PERM;
700                 } else if (mm_flags & AV_CPU_FLAG_MMXEXT) {
701                     c->idct_put              = ff_idct_xvid_mmxext_put;
702                     c->idct_add              = ff_idct_xvid_mmxext_add;
703                     c->idct                  = ff_idct_xvid_mmxext;
704                 } else {
705                     c->idct_put              = ff_idct_xvid_mmx_put;
706                     c->idct_add              = ff_idct_xvid_mmx_add;
707                     c->idct                  = ff_idct_xvid_mmx;
708                 }
709             }
710         }
711 #endif /* HAVE_INLINE_ASM */
712
713         dsputil_init_mmx(c, avctx, mm_flags);
714     }
715
716     if (mm_flags & AV_CPU_FLAG_MMXEXT)
717         dsputil_init_mmxext(c, avctx, mm_flags);
718
719     if (mm_flags & AV_CPU_FLAG_SSE)
720         dsputil_init_sse(c, avctx, mm_flags);
721
722     if (mm_flags & AV_CPU_FLAG_SSE2)
723         dsputil_init_sse2(c, avctx, mm_flags);
724
725     if (mm_flags & AV_CPU_FLAG_SSSE3)
726         dsputil_init_ssse3(c, avctx, mm_flags);
727
728     if (mm_flags & AV_CPU_FLAG_SSE4)
729         dsputil_init_sse4(c, avctx, mm_flags);
730
731     if (CONFIG_ENCODERS)
732         ff_dsputilenc_init_mmx(c, avctx);
733 }