dsputil: Split off HuffYUV decoding bits into their own context
[ffmpeg.git] / libavcodec / x86 / dsputil_init.c
1 /*
2  * This file is part of Libav.
3  *
4  * Libav is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * Libav is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with Libav; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18
19 #include "config.h"
20 #include "libavutil/attributes.h"
21 #include "libavutil/cpu.h"
22 #include "libavutil/internal.h"
23 #include "libavutil/x86/cpu.h"
24 #include "libavcodec/avcodec.h"
25 #include "libavcodec/dsputil.h"
26 #include "libavcodec/pixels.h"
27 #include "libavcodec/simple_idct.h"
28 #include "libavcodec/version.h"
29 #include "dsputil_x86.h"
30 #include "fpel.h"
31 #include "idct_xvid.h"
32
33 void ff_put_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
34                               int dstStride, int src1Stride, int h);
35 void ff_put_no_rnd_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1,
36                                      uint8_t *src2, int dstStride,
37                                      int src1Stride, int h);
38 void ff_avg_pixels8_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
39                               int dstStride, int src1Stride, int h);
40 void ff_put_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
41                                int dstStride, int src1Stride, int h);
42 void ff_avg_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
43                                int dstStride, int src1Stride, int h);
44 void ff_put_no_rnd_pixels16_l2_mmxext(uint8_t *dst, uint8_t *src1, uint8_t *src2,
45                                       int dstStride, int src1Stride, int h);
46 void ff_put_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
47                                           int dstStride, int srcStride, int h);
48 void ff_avg_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
49                                           int dstStride, int srcStride, int h);
50 void ff_put_no_rnd_mpeg4_qpel16_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
51                                                  int dstStride, int srcStride,
52                                                  int h);
53 void ff_put_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
54                                          int dstStride, int srcStride, int h);
55 void ff_avg_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
56                                          int dstStride, int srcStride, int h);
57 void ff_put_no_rnd_mpeg4_qpel8_h_lowpass_mmxext(uint8_t *dst, uint8_t *src,
58                                                 int dstStride, int srcStride,
59                                                 int h);
60 void ff_put_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
61                                           int dstStride, int srcStride);
62 void ff_avg_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
63                                           int dstStride, int srcStride);
64 void ff_put_no_rnd_mpeg4_qpel16_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
65                                                  int dstStride, int srcStride);
66 void ff_put_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
67                                          int dstStride, int srcStride);
68 void ff_avg_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
69                                          int dstStride, int srcStride);
70 void ff_put_no_rnd_mpeg4_qpel8_v_lowpass_mmxext(uint8_t *dst, uint8_t *src,
71                                                 int dstStride, int srcStride);
72 #define ff_put_no_rnd_pixels16_mmxext ff_put_pixels16_mmxext
73 #define ff_put_no_rnd_pixels8_mmxext ff_put_pixels8_mmxext
74
75 int32_t ff_scalarproduct_int16_mmxext(const int16_t *v1, const int16_t *v2,
76                                       int order);
77 int32_t ff_scalarproduct_int16_sse2(const int16_t *v1, const int16_t *v2,
78                                     int order);
79 int32_t ff_scalarproduct_and_madd_int16_mmxext(int16_t *v1, const int16_t *v2,
80                                                const int16_t *v3,
81                                                int order, int mul);
82 int32_t ff_scalarproduct_and_madd_int16_sse2(int16_t *v1, const int16_t *v2,
83                                              const int16_t *v3,
84                                              int order, int mul);
85 int32_t ff_scalarproduct_and_madd_int16_ssse3(int16_t *v1, const int16_t *v2,
86                                               const int16_t *v3,
87                                               int order, int mul);
88
89 void ff_bswap32_buf_ssse3(uint32_t *dst, const uint32_t *src, int w);
90 void ff_bswap32_buf_sse2(uint32_t *dst, const uint32_t *src, int w);
91
92 void ff_vector_clip_int32_mmx(int32_t *dst, const int32_t *src,
93                               int32_t min, int32_t max, unsigned int len);
94 void ff_vector_clip_int32_sse2(int32_t *dst, const int32_t *src,
95                                int32_t min, int32_t max, unsigned int len);
96 void ff_vector_clip_int32_int_sse2(int32_t *dst, const int32_t *src,
97                                    int32_t min, int32_t max, unsigned int len);
98 void ff_vector_clip_int32_sse4(int32_t *dst, const int32_t *src,
99                                int32_t min, int32_t max, unsigned int len);
100
101 #if HAVE_YASM
102
103 CALL_2X_PIXELS(ff_avg_pixels16_mmxext, ff_avg_pixels8_mmxext, 8)
104 CALL_2X_PIXELS(ff_put_pixels16_mmxext, ff_put_pixels8_mmxext, 8)
105
106 #define QPEL_OP(OPNAME, RND, MMX)                                       \
107 static void OPNAME ## qpel8_mc00_ ## MMX(uint8_t *dst, uint8_t *src,    \
108                                          ptrdiff_t stride)              \
109 {                                                                       \
110     ff_ ## OPNAME ## pixels8_ ## MMX(dst, src, stride, 8);              \
111 }                                                                       \
112                                                                         \
113 static void OPNAME ## qpel8_mc10_ ## MMX(uint8_t *dst, uint8_t *src,    \
114                                          ptrdiff_t stride)              \
115 {                                                                       \
116     uint64_t temp[8];                                                   \
117     uint8_t *const half = (uint8_t *) temp;                             \
118     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
119                                                    stride, 8);          \
120     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
121                                         stride, stride, 8);             \
122 }                                                                       \
123                                                                         \
124 static void OPNAME ## qpel8_mc20_ ## MMX(uint8_t *dst, uint8_t *src,    \
125                                          ptrdiff_t stride)              \
126 {                                                                       \
127     ff_ ## OPNAME ## mpeg4_qpel8_h_lowpass_ ## MMX(dst, src, stride,    \
128                                                    stride, 8);          \
129 }                                                                       \
130                                                                         \
131 static void OPNAME ## qpel8_mc30_ ## MMX(uint8_t *dst, uint8_t *src,    \
132                                          ptrdiff_t stride)              \
133 {                                                                       \
134     uint64_t temp[8];                                                   \
135     uint8_t *const half = (uint8_t *) temp;                             \
136     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(half, src, 8,        \
137                                                    stride, 8);          \
138     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + 1, half, stride,     \
139                                         stride, 8);                     \
140 }                                                                       \
141                                                                         \
142 static void OPNAME ## qpel8_mc01_ ## MMX(uint8_t *dst, uint8_t *src,    \
143                                          ptrdiff_t stride)              \
144 {                                                                       \
145     uint64_t temp[8];                                                   \
146     uint8_t *const half = (uint8_t *) temp;                             \
147     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
148                                                    8, stride);          \
149     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src, half,                 \
150                                         stride, stride, 8);             \
151 }                                                                       \
152                                                                         \
153 static void OPNAME ## qpel8_mc02_ ## MMX(uint8_t *dst, uint8_t *src,    \
154                                          ptrdiff_t stride)              \
155 {                                                                       \
156     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, src,            \
157                                                    stride, stride);     \
158 }                                                                       \
159                                                                         \
160 static void OPNAME ## qpel8_mc03_ ## MMX(uint8_t *dst, uint8_t *src,    \
161                                          ptrdiff_t stride)              \
162 {                                                                       \
163     uint64_t temp[8];                                                   \
164     uint8_t *const half = (uint8_t *) temp;                             \
165     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(half, src,           \
166                                                    8, stride);          \
167     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, src + stride, half, stride,\
168                                         stride, 8);                     \
169 }                                                                       \
170                                                                         \
171 static void OPNAME ## qpel8_mc11_ ## MMX(uint8_t *dst, uint8_t *src,    \
172                                          ptrdiff_t stride)              \
173 {                                                                       \
174     uint64_t half[8 + 9];                                               \
175     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
176     uint8_t *const halfHV = (uint8_t *) half;                           \
177     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
178                                                    stride, 9);          \
179     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
180                                         stride, 9);                     \
181     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
182     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
183                                         stride, 8, 8);                  \
184 }                                                                       \
185                                                                         \
186 static void OPNAME ## qpel8_mc31_ ## MMX(uint8_t *dst, uint8_t *src,    \
187                                          ptrdiff_t stride)              \
188 {                                                                       \
189     uint64_t half[8 + 9];                                               \
190     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
191     uint8_t *const halfHV = (uint8_t *) half;                           \
192     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
193                                                    stride, 9);          \
194     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
195                                         stride, 9);                     \
196     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
197     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
198                                         stride, 8, 8);                  \
199 }                                                                       \
200                                                                         \
201 static void OPNAME ## qpel8_mc13_ ## MMX(uint8_t *dst, uint8_t *src,    \
202                                          ptrdiff_t stride)              \
203 {                                                                       \
204     uint64_t half[8 + 9];                                               \
205     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
206     uint8_t *const halfHV = (uint8_t *) half;                           \
207     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
208                                                    stride, 9);          \
209     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH, 8,           \
210                                         stride, 9);                     \
211     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
212     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
213                                         stride, 8, 8);                  \
214 }                                                                       \
215                                                                         \
216 static void OPNAME ## qpel8_mc33_ ## MMX(uint8_t *dst, uint8_t *src,    \
217                                          ptrdiff_t stride)              \
218 {                                                                       \
219     uint64_t half[8 + 9];                                               \
220     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
221     uint8_t *const halfHV = (uint8_t *) half;                           \
222     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
223                                                    stride, 9);          \
224     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
225                                         stride, 9);                     \
226     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
227     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
228                                         stride, 8, 8);                  \
229 }                                                                       \
230                                                                         \
231 static void OPNAME ## qpel8_mc21_ ## MMX(uint8_t *dst, uint8_t *src,    \
232                                          ptrdiff_t stride)              \
233 {                                                                       \
234     uint64_t half[8 + 9];                                               \
235     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
236     uint8_t *const halfHV = (uint8_t *) half;                           \
237     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
238                                                    stride, 9);          \
239     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
240     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH, halfHV,             \
241                                         stride, 8, 8);                  \
242 }                                                                       \
243                                                                         \
244 static void OPNAME ## qpel8_mc23_ ## MMX(uint8_t *dst, uint8_t *src,    \
245                                          ptrdiff_t stride)              \
246 {                                                                       \
247     uint64_t half[8 + 9];                                               \
248     uint8_t *const halfH  = (uint8_t *) half + 64;                      \
249     uint8_t *const halfHV = (uint8_t *) half;                           \
250     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
251                                                    stride, 9);          \
252     ff_put ## RND ## mpeg4_qpel8_v_lowpass_ ## MMX(halfHV, halfH, 8, 8);\
253     ff_ ## OPNAME ## pixels8_l2_ ## MMX(dst, halfH + 8, halfHV,         \
254                                         stride, 8, 8);                  \
255 }                                                                       \
256                                                                         \
257 static void OPNAME ## qpel8_mc12_ ## MMX(uint8_t *dst, uint8_t *src,    \
258                                          ptrdiff_t stride)              \
259 {                                                                       \
260     uint64_t half[8 + 9];                                               \
261     uint8_t *const halfH = (uint8_t *) half;                            \
262     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
263                                                    stride, 9);          \
264     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src, halfH,              \
265                                         8, stride, 9);                  \
266     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
267                                                    stride, 8);          \
268 }                                                                       \
269                                                                         \
270 static void OPNAME ## qpel8_mc32_ ## MMX(uint8_t *dst, uint8_t *src,    \
271                                          ptrdiff_t stride)              \
272 {                                                                       \
273     uint64_t half[8 + 9];                                               \
274     uint8_t *const halfH = (uint8_t *) half;                            \
275     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
276                                                    stride, 9);          \
277     ff_put ## RND ## pixels8_l2_ ## MMX(halfH, src + 1, halfH, 8,       \
278                                         stride, 9);                     \
279     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
280                                                    stride, 8);          \
281 }                                                                       \
282                                                                         \
283 static void OPNAME ## qpel8_mc22_ ## MMX(uint8_t *dst, uint8_t *src,    \
284                                          ptrdiff_t stride)              \
285 {                                                                       \
286     uint64_t half[9];                                                   \
287     uint8_t *const halfH = (uint8_t *) half;                            \
288     ff_put ## RND ## mpeg4_qpel8_h_lowpass_ ## MMX(halfH, src, 8,       \
289                                                    stride, 9);          \
290     ff_ ## OPNAME ## mpeg4_qpel8_v_lowpass_ ## MMX(dst, halfH,          \
291                                                    stride, 8);          \
292 }                                                                       \
293                                                                         \
294 static void OPNAME ## qpel16_mc00_ ## MMX(uint8_t *dst, uint8_t *src,   \
295                                           ptrdiff_t stride)             \
296 {                                                                       \
297     ff_ ## OPNAME ## pixels16_ ## MMX(dst, src, stride, 16);            \
298 }                                                                       \
299                                                                         \
300 static void OPNAME ## qpel16_mc10_ ## MMX(uint8_t *dst, uint8_t *src,   \
301                                           ptrdiff_t stride)             \
302 {                                                                       \
303     uint64_t temp[32];                                                  \
304     uint8_t *const half = (uint8_t *) temp;                             \
305     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
306                                                     stride, 16);        \
307     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
308                                          stride, 16);                   \
309 }                                                                       \
310                                                                         \
311 static void OPNAME ## qpel16_mc20_ ## MMX(uint8_t *dst, uint8_t *src,   \
312                                           ptrdiff_t stride)             \
313 {                                                                       \
314     ff_ ## OPNAME ## mpeg4_qpel16_h_lowpass_ ## MMX(dst, src,           \
315                                                     stride, stride, 16);\
316 }                                                                       \
317                                                                         \
318 static void OPNAME ## qpel16_mc30_ ## MMX(uint8_t *dst, uint8_t *src,   \
319                                           ptrdiff_t stride)             \
320 {                                                                       \
321     uint64_t temp[32];                                                  \
322     uint8_t *const half = (uint8_t*) temp;                              \
323     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(half, src, 16,      \
324                                                     stride, 16);        \
325     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src + 1, half,            \
326                                          stride, stride, 16);           \
327 }                                                                       \
328                                                                         \
329 static void OPNAME ## qpel16_mc01_ ## MMX(uint8_t *dst, uint8_t *src,   \
330                                           ptrdiff_t stride)             \
331 {                                                                       \
332     uint64_t temp[32];                                                  \
333     uint8_t *const half = (uint8_t *) temp;                             \
334     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
335                                                     stride);            \
336     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src, half, stride,        \
337                                          stride, 16);                   \
338 }                                                                       \
339                                                                         \
340 static void OPNAME ## qpel16_mc02_ ## MMX(uint8_t *dst, uint8_t *src,   \
341                                           ptrdiff_t stride)             \
342 {                                                                       \
343     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, src,           \
344                                                     stride, stride);    \
345 }                                                                       \
346                                                                         \
347 static void OPNAME ## qpel16_mc03_ ## MMX(uint8_t *dst, uint8_t *src,   \
348                                           ptrdiff_t stride)             \
349 {                                                                       \
350     uint64_t temp[32];                                                  \
351     uint8_t *const half = (uint8_t *) temp;                             \
352     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(half, src, 16,      \
353                                                     stride);            \
354     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, src+stride, half,         \
355                                          stride, stride, 16);           \
356 }                                                                       \
357                                                                         \
358 static void OPNAME ## qpel16_mc11_ ## MMX(uint8_t *dst, uint8_t *src,   \
359                                           ptrdiff_t stride)             \
360 {                                                                       \
361     uint64_t half[16 * 2 + 17 * 2];                                     \
362     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
363     uint8_t *const halfHV = (uint8_t *) half;                           \
364     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
365                                                     stride, 17);        \
366     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
367                                          stride, 17);                   \
368     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
369                                                     16, 16);            \
370     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
371                                          stride, 16, 16);               \
372 }                                                                       \
373                                                                         \
374 static void OPNAME ## qpel16_mc31_ ## MMX(uint8_t *dst, uint8_t *src,   \
375                                           ptrdiff_t stride)             \
376 {                                                                       \
377     uint64_t half[16 * 2 + 17 * 2];                                     \
378     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
379     uint8_t *const halfHV = (uint8_t *) half;                           \
380     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
381                                                     stride, 17);        \
382     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
383                                          stride, 17);                   \
384     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
385                                                     16, 16);            \
386     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
387                                          stride, 16, 16);               \
388 }                                                                       \
389                                                                         \
390 static void OPNAME ## qpel16_mc13_ ## MMX(uint8_t *dst, uint8_t *src,   \
391                                           ptrdiff_t stride)             \
392 {                                                                       \
393     uint64_t half[16 * 2 + 17 * 2];                                     \
394     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
395     uint8_t *const halfHV = (uint8_t *) half;                           \
396     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
397                                                     stride, 17);        \
398     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
399                                          stride, 17);                   \
400     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
401                                                     16, 16);            \
402     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
403                                          stride, 16, 16);               \
404 }                                                                       \
405                                                                         \
406 static void OPNAME ## qpel16_mc33_ ## MMX(uint8_t *dst, uint8_t *src,   \
407                                           ptrdiff_t stride)             \
408 {                                                                       \
409     uint64_t half[16 * 2 + 17 * 2];                                     \
410     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
411     uint8_t *const halfHV = (uint8_t *) half;                           \
412     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
413                                                     stride, 17);        \
414     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
415                                          stride, 17);                   \
416     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
417                                                     16, 16);            \
418     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
419                                          stride, 16, 16);               \
420 }                                                                       \
421                                                                         \
422 static void OPNAME ## qpel16_mc21_ ## MMX(uint8_t *dst, uint8_t *src,   \
423                                           ptrdiff_t stride)             \
424 {                                                                       \
425     uint64_t half[16 * 2 + 17 * 2];                                     \
426     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
427     uint8_t *const halfHV = (uint8_t *) half;                           \
428     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
429                                                     stride, 17);        \
430     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
431                                                     16, 16);            \
432     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH, halfHV,            \
433                                          stride, 16, 16);               \
434 }                                                                       \
435                                                                         \
436 static void OPNAME ## qpel16_mc23_ ## MMX(uint8_t *dst, uint8_t *src,   \
437                                           ptrdiff_t stride)             \
438 {                                                                       \
439     uint64_t half[16 * 2 + 17 * 2];                                     \
440     uint8_t *const halfH  = (uint8_t *) half + 256;                     \
441     uint8_t *const halfHV = (uint8_t *) half;                           \
442     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
443                                                     stride, 17);        \
444     ff_put ## RND ## mpeg4_qpel16_v_lowpass_ ## MMX(halfHV, halfH,      \
445                                                     16, 16);            \
446     ff_ ## OPNAME ## pixels16_l2_ ## MMX(dst, halfH + 16, halfHV,       \
447                                          stride, 16, 16);               \
448 }                                                                       \
449                                                                         \
450 static void OPNAME ## qpel16_mc12_ ## MMX(uint8_t *dst, uint8_t *src,   \
451                                           ptrdiff_t stride)             \
452 {                                                                       \
453     uint64_t half[17 * 2];                                              \
454     uint8_t *const halfH = (uint8_t *) half;                            \
455     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
456                                                     stride, 17);        \
457     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src, halfH, 16,         \
458                                          stride, 17);                   \
459     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
460                                                     stride, 16);        \
461 }                                                                       \
462                                                                         \
463 static void OPNAME ## qpel16_mc32_ ## MMX(uint8_t *dst, uint8_t *src,   \
464                                           ptrdiff_t stride)             \
465 {                                                                       \
466     uint64_t half[17 * 2];                                              \
467     uint8_t *const halfH = (uint8_t *) half;                            \
468     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
469                                                     stride, 17);        \
470     ff_put ## RND ## pixels16_l2_ ## MMX(halfH, src + 1, halfH, 16,     \
471                                          stride, 17);                   \
472     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
473                                                     stride, 16);        \
474 }                                                                       \
475                                                                         \
476 static void OPNAME ## qpel16_mc22_ ## MMX(uint8_t *dst, uint8_t *src,   \
477                                           ptrdiff_t stride)             \
478 {                                                                       \
479     uint64_t half[17 * 2];                                              \
480     uint8_t *const halfH = (uint8_t *) half;                            \
481     ff_put ## RND ## mpeg4_qpel16_h_lowpass_ ## MMX(halfH, src, 16,     \
482                                                     stride, 17);        \
483     ff_ ## OPNAME ## mpeg4_qpel16_v_lowpass_ ## MMX(dst, halfH,         \
484                                                     stride, 16);        \
485 }
486
487 QPEL_OP(put_,        _,        mmxext)
488 QPEL_OP(avg_,        _,        mmxext)
489 QPEL_OP(put_no_rnd_, _no_rnd_, mmxext)
490
491 #endif /* HAVE_YASM */
492
493 #define SET_QPEL_FUNCS(PFX, IDX, SIZE, CPU, PREFIX)                          \
494 do {                                                                         \
495     c->PFX ## _pixels_tab[IDX][ 0] = PREFIX ## PFX ## SIZE ## _mc00_ ## CPU; \
496     c->PFX ## _pixels_tab[IDX][ 1] = PREFIX ## PFX ## SIZE ## _mc10_ ## CPU; \
497     c->PFX ## _pixels_tab[IDX][ 2] = PREFIX ## PFX ## SIZE ## _mc20_ ## CPU; \
498     c->PFX ## _pixels_tab[IDX][ 3] = PREFIX ## PFX ## SIZE ## _mc30_ ## CPU; \
499     c->PFX ## _pixels_tab[IDX][ 4] = PREFIX ## PFX ## SIZE ## _mc01_ ## CPU; \
500     c->PFX ## _pixels_tab[IDX][ 5] = PREFIX ## PFX ## SIZE ## _mc11_ ## CPU; \
501     c->PFX ## _pixels_tab[IDX][ 6] = PREFIX ## PFX ## SIZE ## _mc21_ ## CPU; \
502     c->PFX ## _pixels_tab[IDX][ 7] = PREFIX ## PFX ## SIZE ## _mc31_ ## CPU; \
503     c->PFX ## _pixels_tab[IDX][ 8] = PREFIX ## PFX ## SIZE ## _mc02_ ## CPU; \
504     c->PFX ## _pixels_tab[IDX][ 9] = PREFIX ## PFX ## SIZE ## _mc12_ ## CPU; \
505     c->PFX ## _pixels_tab[IDX][10] = PREFIX ## PFX ## SIZE ## _mc22_ ## CPU; \
506     c->PFX ## _pixels_tab[IDX][11] = PREFIX ## PFX ## SIZE ## _mc32_ ## CPU; \
507     c->PFX ## _pixels_tab[IDX][12] = PREFIX ## PFX ## SIZE ## _mc03_ ## CPU; \
508     c->PFX ## _pixels_tab[IDX][13] = PREFIX ## PFX ## SIZE ## _mc13_ ## CPU; \
509     c->PFX ## _pixels_tab[IDX][14] = PREFIX ## PFX ## SIZE ## _mc23_ ## CPU; \
510     c->PFX ## _pixels_tab[IDX][15] = PREFIX ## PFX ## SIZE ## _mc33_ ## CPU; \
511 } while (0)
512
513 static av_cold void dsputil_init_mmx(DSPContext *c, AVCodecContext *avctx,
514                                      int cpu_flags, unsigned high_bit_depth)
515 {
516 #if HAVE_MMX_INLINE
517     c->put_pixels_clamped        = ff_put_pixels_clamped_mmx;
518     c->put_signed_pixels_clamped = ff_put_signed_pixels_clamped_mmx;
519     c->add_pixels_clamped        = ff_add_pixels_clamped_mmx;
520
521     if (!high_bit_depth) {
522         c->clear_block  = ff_clear_block_mmx;
523         c->clear_blocks = ff_clear_blocks_mmx;
524         c->draw_edges   = ff_draw_edges_mmx;
525
526         switch (avctx->idct_algo) {
527         case FF_IDCT_AUTO:
528         case FF_IDCT_SIMPLEMMX:
529             c->idct_put              = ff_simple_idct_put_mmx;
530             c->idct_add              = ff_simple_idct_add_mmx;
531             c->idct                  = ff_simple_idct_mmx;
532             c->idct_permutation_type = FF_SIMPLE_IDCT_PERM;
533             break;
534         case FF_IDCT_XVIDMMX:
535             c->idct_put              = ff_idct_xvid_mmx_put;
536             c->idct_add              = ff_idct_xvid_mmx_add;
537             c->idct                  = ff_idct_xvid_mmx;
538             break;
539         }
540     }
541
542     c->gmc = ff_gmc_mmx;
543 #endif /* HAVE_MMX_INLINE */
544
545 #if HAVE_MMX_EXTERNAL
546     c->vector_clip_int32 = ff_vector_clip_int32_mmx;
547 #endif /* HAVE_MMX_EXTERNAL */
548 }
549
550 static av_cold void dsputil_init_mmxext(DSPContext *c, AVCodecContext *avctx,
551                                         int cpu_flags, unsigned high_bit_depth)
552 {
553 #if HAVE_MMXEXT_INLINE
554     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
555         c->idct_put = ff_idct_xvid_mmxext_put;
556         c->idct_add = ff_idct_xvid_mmxext_add;
557         c->idct     = ff_idct_xvid_mmxext;
558     }
559 #endif /* HAVE_MMXEXT_INLINE */
560
561 #if HAVE_MMXEXT_EXTERNAL
562     SET_QPEL_FUNCS(avg_qpel,        0, 16, mmxext, );
563     SET_QPEL_FUNCS(avg_qpel,        1,  8, mmxext, );
564
565     SET_QPEL_FUNCS(put_qpel,        0, 16, mmxext, );
566     SET_QPEL_FUNCS(put_qpel,        1,  8, mmxext, );
567     SET_QPEL_FUNCS(put_no_rnd_qpel, 0, 16, mmxext, );
568     SET_QPEL_FUNCS(put_no_rnd_qpel, 1,  8, mmxext, );
569
570     c->scalarproduct_int16          = ff_scalarproduct_int16_mmxext;
571     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_mmxext;
572 #endif /* HAVE_MMXEXT_EXTERNAL */
573 }
574
575 static av_cold void dsputil_init_sse(DSPContext *c, AVCodecContext *avctx,
576                                      int cpu_flags, unsigned high_bit_depth)
577 {
578 #if HAVE_SSE_INLINE
579     c->vector_clipf = ff_vector_clipf_sse;
580
581 #if FF_API_XVMC
582 FF_DISABLE_DEPRECATION_WARNINGS
583     /* XvMCCreateBlocks() may not allocate 16-byte aligned blocks */
584     if (CONFIG_MPEG_XVMC_DECODER && avctx->xvmc_acceleration > 1)
585         return;
586 FF_ENABLE_DEPRECATION_WARNINGS
587 #endif /* FF_API_XVMC */
588
589     if (!high_bit_depth) {
590         c->clear_block  = ff_clear_block_sse;
591         c->clear_blocks = ff_clear_blocks_sse;
592     }
593 #endif /* HAVE_SSE_INLINE */
594 }
595
596 static av_cold void dsputil_init_sse2(DSPContext *c, AVCodecContext *avctx,
597                                       int cpu_flags, unsigned high_bit_depth)
598 {
599 #if HAVE_SSE2_INLINE
600     if (!high_bit_depth && avctx->idct_algo == FF_IDCT_XVIDMMX) {
601         c->idct_put              = ff_idct_xvid_sse2_put;
602         c->idct_add              = ff_idct_xvid_sse2_add;
603         c->idct                  = ff_idct_xvid_sse2;
604         c->idct_permutation_type = FF_SSE2_IDCT_PERM;
605     }
606 #endif /* HAVE_SSE2_INLINE */
607
608 #if HAVE_SSE2_EXTERNAL
609     c->scalarproduct_int16          = ff_scalarproduct_int16_sse2;
610     c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_sse2;
611     if (cpu_flags & AV_CPU_FLAG_ATOM) {
612         c->vector_clip_int32 = ff_vector_clip_int32_int_sse2;
613     } else {
614         c->vector_clip_int32 = ff_vector_clip_int32_sse2;
615     }
616     c->bswap_buf = ff_bswap32_buf_sse2;
617 #endif /* HAVE_SSE2_EXTERNAL */
618 }
619
620 static av_cold void dsputil_init_ssse3(DSPContext *c, AVCodecContext *avctx,
621                                        int cpu_flags, unsigned high_bit_depth)
622 {
623 #if HAVE_SSSE3_EXTERNAL
624     if (!(cpu_flags & (AV_CPU_FLAG_SSE42 | AV_CPU_FLAG_3DNOW))) // cachesplit
625         c->scalarproduct_and_madd_int16 = ff_scalarproduct_and_madd_int16_ssse3;
626     c->bswap_buf = ff_bswap32_buf_ssse3;
627 #endif /* HAVE_SSSE3_EXTERNAL */
628 }
629
630 static av_cold void dsputil_init_sse4(DSPContext *c, AVCodecContext *avctx,
631                                       int cpu_flags, unsigned high_bit_depth)
632 {
633 #if HAVE_SSE4_EXTERNAL
634     c->vector_clip_int32 = ff_vector_clip_int32_sse4;
635 #endif /* HAVE_SSE4_EXTERNAL */
636 }
637
638 av_cold void ff_dsputil_init_x86(DSPContext *c, AVCodecContext *avctx,
639                                  unsigned high_bit_depth)
640 {
641     int cpu_flags = av_get_cpu_flags();
642
643     if (X86_MMX(cpu_flags))
644         dsputil_init_mmx(c, avctx, cpu_flags, high_bit_depth);
645
646     if (X86_MMXEXT(cpu_flags))
647         dsputil_init_mmxext(c, avctx, cpu_flags, high_bit_depth);
648
649     if (X86_SSE(cpu_flags))
650         dsputil_init_sse(c, avctx, cpu_flags, high_bit_depth);
651
652     if (X86_SSE2(cpu_flags))
653         dsputil_init_sse2(c, avctx, cpu_flags, high_bit_depth);
654
655     if (EXTERNAL_SSSE3(cpu_flags))
656         dsputil_init_ssse3(c, avctx, cpu_flags, high_bit_depth);
657
658     if (EXTERNAL_SSE4(cpu_flags))
659         dsputil_init_sse4(c, avctx, cpu_flags, high_bit_depth);
660
661     if (CONFIG_ENCODERS)
662         ff_dsputilenc_init_mmx(c, avctx, high_bit_depth);
663 }