swscale: fix compile on x86-32.
[ffmpeg.git] / libswscale / x86 / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of Libav.
5  *
6  * Libav is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * Libav is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with Libav; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #include "swscale_template.h"
22
23 #undef REAL_MOVNTQ
24 #undef MOVNTQ
25 #undef PREFETCH
26
27 #if COMPILE_TEMPLATE_MMX2
28 #define PREFETCH "prefetchnta"
29 #else
30 #define PREFETCH  " # nop"
31 #endif
32
33 #if COMPILE_TEMPLATE_MMX2
34 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
35 #else
36 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
37 #endif
38 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
39
40 #define YSCALEYUV2YV12X(offset, dest, end, pos) \
41     __asm__ volatile(\
42         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
43         "movq                             %%mm3, %%mm4      \n\t"\
44         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
45         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
46         ".p2align                             4             \n\t" /* FIXME Unroll? */\
47         "1:                                                 \n\t"\
48         "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\
49         "movq                (%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\
50         "movq               8(%%"REG_S", %3, 2), %%mm5      \n\t" /* srcData */\
51         "add                                $16, %%"REG_d"  \n\t"\
52         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
53         "test                         %%"REG_S", %%"REG_S"  \n\t"\
54         "pmulhw                           %%mm0, %%mm2      \n\t"\
55         "pmulhw                           %%mm0, %%mm5      \n\t"\
56         "paddw                            %%mm2, %%mm3      \n\t"\
57         "paddw                            %%mm5, %%mm4      \n\t"\
58         " jnz                                1b             \n\t"\
59         "psraw                               $3, %%mm3      \n\t"\
60         "psraw                               $3, %%mm4      \n\t"\
61         "packuswb                         %%mm4, %%mm3      \n\t"\
62         MOVNTQ(%%mm3, (%1, %%REGa))\
63         "add                                 $8, %3         \n\t"\
64         "cmp                                 %2, %3         \n\t"\
65         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
66         "movq                             %%mm3, %%mm4      \n\t"\
67         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
68         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
69         "jb                                  1b             \n\t"\
70         :: "r" (&c->redDither),\
71            "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\
72         : "%"REG_d, "%"REG_S\
73     );
74
75 static inline void RENAME(yuv2yuvX)(SwsContext *c, const int16_t *lumFilter,
76                                     const int16_t **lumSrc, int lumFilterSize,
77                                     const int16_t *chrFilter, const int16_t **chrUSrc,
78                                     const int16_t **chrVSrc,
79                                     int chrFilterSize, const int16_t **alpSrc,
80                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
81                                     uint8_t *aDest, long dstW, long chrDstW)
82 {
83     if (uDest) {
84         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)
85         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, vDest, chrDstW + c->uv_off, c->uv_off)
86     }
87     if (CONFIG_SWSCALE_ALPHA && aDest) {
88         YSCALEYUV2YV12X(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)
89     }
90
91     YSCALEYUV2YV12X(LUM_MMX_FILTER_OFFSET, dest, dstW, 0)
92 }
93
94 #define YSCALEYUV2YV12X_ACCURATE(offset, dest, end, pos) \
95     __asm__ volatile(\
96         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
97         "pxor                             %%mm4, %%mm4      \n\t"\
98         "pxor                             %%mm5, %%mm5      \n\t"\
99         "pxor                             %%mm6, %%mm6      \n\t"\
100         "pxor                             %%mm7, %%mm7      \n\t"\
101         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
102         ".p2align                             4             \n\t"\
103         "1:                                                 \n\t"\
104         "movq                (%%"REG_S", %3, 2), %%mm0      \n\t" /* srcData */\
105         "movq               8(%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\
106         "mov        "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"  \n\t"\
107         "movq                (%%"REG_S", %3, 2), %%mm1      \n\t" /* srcData */\
108         "movq                             %%mm0, %%mm3      \n\t"\
109         "punpcklwd                        %%mm1, %%mm0      \n\t"\
110         "punpckhwd                        %%mm1, %%mm3      \n\t"\
111         "movq       "STR(APCK_COEF)"(%%"REG_d"), %%mm1      \n\t" /* filterCoeff */\
112         "pmaddwd                          %%mm1, %%mm0      \n\t"\
113         "pmaddwd                          %%mm1, %%mm3      \n\t"\
114         "paddd                            %%mm0, %%mm4      \n\t"\
115         "paddd                            %%mm3, %%mm5      \n\t"\
116         "movq               8(%%"REG_S", %3, 2), %%mm3      \n\t" /* srcData */\
117         "mov        "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"  \n\t"\
118         "add                  $"STR(APCK_SIZE)", %%"REG_d"  \n\t"\
119         "test                         %%"REG_S", %%"REG_S"  \n\t"\
120         "movq                             %%mm2, %%mm0      \n\t"\
121         "punpcklwd                        %%mm3, %%mm2      \n\t"\
122         "punpckhwd                        %%mm3, %%mm0      \n\t"\
123         "pmaddwd                          %%mm1, %%mm2      \n\t"\
124         "pmaddwd                          %%mm1, %%mm0      \n\t"\
125         "paddd                            %%mm2, %%mm6      \n\t"\
126         "paddd                            %%mm0, %%mm7      \n\t"\
127         " jnz                                1b             \n\t"\
128         "psrad                              $16, %%mm4      \n\t"\
129         "psrad                              $16, %%mm5      \n\t"\
130         "psrad                              $16, %%mm6      \n\t"\
131         "psrad                              $16, %%mm7      \n\t"\
132         "movq             "VROUNDER_OFFSET"(%0), %%mm0      \n\t"\
133         "packssdw                         %%mm5, %%mm4      \n\t"\
134         "packssdw                         %%mm7, %%mm6      \n\t"\
135         "paddw                            %%mm0, %%mm4      \n\t"\
136         "paddw                            %%mm0, %%mm6      \n\t"\
137         "psraw                               $3, %%mm4      \n\t"\
138         "psraw                               $3, %%mm6      \n\t"\
139         "packuswb                         %%mm6, %%mm4      \n\t"\
140         MOVNTQ(%%mm4, (%1, %%REGa))\
141         "add                                 $8, %3         \n\t"\
142         "cmp                                 %2, %3         \n\t"\
143         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
144         "pxor                             %%mm4, %%mm4      \n\t"\
145         "pxor                             %%mm5, %%mm5      \n\t"\
146         "pxor                             %%mm6, %%mm6      \n\t"\
147         "pxor                             %%mm7, %%mm7      \n\t"\
148         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
149         "jb                                  1b             \n\t"\
150         :: "r" (&c->redDither),\
151         "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\
152         : "%"REG_a, "%"REG_d, "%"REG_S\
153     );
154
155 static inline void RENAME(yuv2yuvX_ar)(SwsContext *c, const int16_t *lumFilter,
156                                        const int16_t **lumSrc, int lumFilterSize,
157                                        const int16_t *chrFilter, const int16_t **chrUSrc,
158                                        const int16_t **chrVSrc,
159                                        int chrFilterSize, const int16_t **alpSrc,
160                                        uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
161                                        uint8_t *aDest, long dstW, long chrDstW)
162 {
163     if (uDest) {
164         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)
165         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, vDest, chrDstW + c->uv_off, c->uv_off)
166     }
167     if (CONFIG_SWSCALE_ALPHA && aDest) {
168         YSCALEYUV2YV12X_ACCURATE(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)
169     }
170
171     YSCALEYUV2YV12X_ACCURATE(LUM_MMX_FILTER_OFFSET, dest, dstW, 0)
172 }
173
174 #define YSCALEYUV2YV121 \
175     "mov %2, %%"REG_a"                    \n\t"\
176     ".p2align               4             \n\t" /* FIXME Unroll? */\
177     "1:                                   \n\t"\
178     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
179     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
180     "psraw                 $7, %%mm0      \n\t"\
181     "psraw                 $7, %%mm1      \n\t"\
182     "packuswb           %%mm1, %%mm0      \n\t"\
183     MOVNTQ(%%mm0, (%1, %%REGa))\
184     "add                   $8, %%"REG_a"  \n\t"\
185     "jnc                   1b             \n\t"
186
187 static inline void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc,
188                                     const int16_t *chrUSrc, const int16_t *chrVSrc,
189                                     const int16_t *alpSrc,
190                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
191                                     uint8_t *aDest, long dstW, long chrDstW)
192 {
193     long p= 4;
194     const uint8_t *src[4]= { alpSrc + dstW, lumSrc + dstW, chrUSrc + chrDstW, chrVSrc + chrDstW };
195     uint8_t *dst[4]= { aDest, dest, uDest, vDest };
196     x86_reg counter[4]= { dstW, dstW, chrDstW, chrDstW };
197
198     while (p--) {
199         if (dst[p]) {
200             __asm__ volatile(
201                YSCALEYUV2YV121
202                :: "r" (src[p]), "r" (dst[p] + counter[p]),
203                   "g" (-counter[p])
204                : "%"REG_a
205             );
206         }
207     }
208 }
209
210 #define YSCALEYUV2YV121_ACCURATE \
211     "mov %2, %%"REG_a"                    \n\t"\
212     "pcmpeqw %%mm7, %%mm7                 \n\t"\
213     "psrlw                 $15, %%mm7     \n\t"\
214     "psllw                  $6, %%mm7     \n\t"\
215     ".p2align                4            \n\t" /* FIXME Unroll? */\
216     "1:                                   \n\t"\
217     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
218     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
219     "paddsw             %%mm7, %%mm0      \n\t"\
220     "paddsw             %%mm7, %%mm1      \n\t"\
221     "psraw                 $7, %%mm0      \n\t"\
222     "psraw                 $7, %%mm1      \n\t"\
223     "packuswb           %%mm1, %%mm0      \n\t"\
224     MOVNTQ(%%mm0, (%1, %%REGa))\
225     "add                   $8, %%"REG_a"  \n\t"\
226     "jnc                   1b             \n\t"
227
228 static inline void RENAME(yuv2yuv1_ar)(SwsContext *c, const int16_t *lumSrc,
229                                        const int16_t *chrUSrc, const int16_t *chrVSrc,
230                                        const int16_t *alpSrc,
231                                        uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
232                                        uint8_t *aDest, long dstW, long chrDstW)
233 {
234     long p= 4;
235     const uint8_t *src[4]= { alpSrc + dstW, lumSrc + dstW, chrUSrc + chrDstW, chrVSrc + chrDstW };
236     uint8_t *dst[4]= { aDest, dest, uDest, vDest };
237     x86_reg counter[4]= { dstW, dstW, chrDstW, chrDstW };
238
239     while (p--) {
240         if (dst[p]) {
241             __asm__ volatile(
242                 YSCALEYUV2YV121_ACCURATE
243                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
244                    "g" (-counter[p])
245                 : "%"REG_a
246             );
247         }
248     }
249 }
250
251 #define YSCALEYUV2PACKEDX_UV \
252     __asm__ volatile(\
253         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
254         ".p2align                      4                \n\t"\
255         "nop                                            \n\t"\
256         "1:                                             \n\t"\
257         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
258         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
259         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
260         "movq                      %%mm3, %%mm4         \n\t"\
261         ".p2align                      4                \n\t"\
262         "2:                                             \n\t"\
263         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
264         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
265         "add                          %6, %%"REG_S"     \n\t" \
266         "movq     (%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
267         "add                         $16, %%"REG_d"     \n\t"\
268         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
269         "pmulhw                    %%mm0, %%mm2         \n\t"\
270         "pmulhw                    %%mm0, %%mm5         \n\t"\
271         "paddw                     %%mm2, %%mm3         \n\t"\
272         "paddw                     %%mm5, %%mm4         \n\t"\
273         "test                  %%"REG_S", %%"REG_S"     \n\t"\
274         " jnz                         2b                \n\t"\
275
276 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
277     "lea                "offset"(%0), %%"REG_d"     \n\t"\
278     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
279     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
280     "movq                    "#dst1", "#dst2"       \n\t"\
281     ".p2align                      4                \n\t"\
282     "2:                                             \n\t"\
283     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
284     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
285     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
286     "add                         $16, %%"REG_d"            \n\t"\
287     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
288     "pmulhw                 "#coeff", "#src1"       \n\t"\
289     "pmulhw                 "#coeff", "#src2"       \n\t"\
290     "paddw                   "#src1", "#dst1"       \n\t"\
291     "paddw                   "#src2", "#dst2"       \n\t"\
292     "test                  %%"REG_S", %%"REG_S"     \n\t"\
293     " jnz                         2b                \n\t"\
294
295 #define YSCALEYUV2PACKEDX \
296     YSCALEYUV2PACKEDX_UV \
297     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
298
299 #define YSCALEYUV2PACKEDX_END                     \
300         :: "r" (&c->redDither),                   \
301             "m" (dummy), "m" (dummy), "m" (dummy),\
302             "r" (dest), "m" (dstW_reg), "m"(uv_off) \
303         : "%"REG_a, "%"REG_d, "%"REG_S            \
304     );
305
306 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
307     __asm__ volatile(\
308         "xor %%"REG_a", %%"REG_a"                       \n\t"\
309         ".p2align                      4                \n\t"\
310         "nop                                            \n\t"\
311         "1:                                             \n\t"\
312         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
313         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
314         "pxor                      %%mm4, %%mm4         \n\t"\
315         "pxor                      %%mm5, %%mm5         \n\t"\
316         "pxor                      %%mm6, %%mm6         \n\t"\
317         "pxor                      %%mm7, %%mm7         \n\t"\
318         ".p2align                      4                \n\t"\
319         "2:                                             \n\t"\
320         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
321         "add                          %6, %%"REG_S"      \n\t" \
322         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
323         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
324         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
325         "movq                      %%mm0, %%mm3         \n\t"\
326         "punpcklwd                 %%mm1, %%mm0         \n\t"\
327         "punpckhwd                 %%mm1, %%mm3         \n\t"\
328         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
329         "pmaddwd                   %%mm1, %%mm0         \n\t"\
330         "pmaddwd                   %%mm1, %%mm3         \n\t"\
331         "paddd                     %%mm0, %%mm4         \n\t"\
332         "paddd                     %%mm3, %%mm5         \n\t"\
333         "add                          %6, %%"REG_S"      \n\t" \
334         "movq     (%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
335         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
336         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
337         "test                  %%"REG_S", %%"REG_S"     \n\t"\
338         "movq                      %%mm2, %%mm0         \n\t"\
339         "punpcklwd                 %%mm3, %%mm2         \n\t"\
340         "punpckhwd                 %%mm3, %%mm0         \n\t"\
341         "pmaddwd                   %%mm1, %%mm2         \n\t"\
342         "pmaddwd                   %%mm1, %%mm0         \n\t"\
343         "paddd                     %%mm2, %%mm6         \n\t"\
344         "paddd                     %%mm0, %%mm7         \n\t"\
345         " jnz                         2b                \n\t"\
346         "psrad                       $16, %%mm4         \n\t"\
347         "psrad                       $16, %%mm5         \n\t"\
348         "psrad                       $16, %%mm6         \n\t"\
349         "psrad                       $16, %%mm7         \n\t"\
350         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
351         "packssdw                  %%mm5, %%mm4         \n\t"\
352         "packssdw                  %%mm7, %%mm6         \n\t"\
353         "paddw                     %%mm0, %%mm4         \n\t"\
354         "paddw                     %%mm0, %%mm6         \n\t"\
355         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
356         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
357
358 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
359     "lea                "offset"(%0), %%"REG_d"     \n\t"\
360     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
361     "pxor                      %%mm1, %%mm1         \n\t"\
362     "pxor                      %%mm5, %%mm5         \n\t"\
363     "pxor                      %%mm7, %%mm7         \n\t"\
364     "pxor                      %%mm6, %%mm6         \n\t"\
365     ".p2align                      4                \n\t"\
366     "2:                                             \n\t"\
367     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
368     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
369     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
370     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
371     "movq                      %%mm0, %%mm3         \n\t"\
372     "punpcklwd                 %%mm4, %%mm0         \n\t"\
373     "punpckhwd                 %%mm4, %%mm3         \n\t"\
374     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
375     "pmaddwd                   %%mm4, %%mm0         \n\t"\
376     "pmaddwd                   %%mm4, %%mm3         \n\t"\
377     "paddd                     %%mm0, %%mm1         \n\t"\
378     "paddd                     %%mm3, %%mm5         \n\t"\
379     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
380     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
381     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
382     "test                  %%"REG_S", %%"REG_S"     \n\t"\
383     "movq                      %%mm2, %%mm0         \n\t"\
384     "punpcklwd                 %%mm3, %%mm2         \n\t"\
385     "punpckhwd                 %%mm3, %%mm0         \n\t"\
386     "pmaddwd                   %%mm4, %%mm2         \n\t"\
387     "pmaddwd                   %%mm4, %%mm0         \n\t"\
388     "paddd                     %%mm2, %%mm7         \n\t"\
389     "paddd                     %%mm0, %%mm6         \n\t"\
390     " jnz                         2b                \n\t"\
391     "psrad                       $16, %%mm1         \n\t"\
392     "psrad                       $16, %%mm5         \n\t"\
393     "psrad                       $16, %%mm7         \n\t"\
394     "psrad                       $16, %%mm6         \n\t"\
395     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
396     "packssdw                  %%mm5, %%mm1         \n\t"\
397     "packssdw                  %%mm6, %%mm7         \n\t"\
398     "paddw                     %%mm0, %%mm1         \n\t"\
399     "paddw                     %%mm0, %%mm7         \n\t"\
400     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
401     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
402
403 #define YSCALEYUV2PACKEDX_ACCURATE \
404     YSCALEYUV2PACKEDX_ACCURATE_UV \
405     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
406
407 #define YSCALEYUV2RGBX \
408     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
409     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
410     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
411     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
412     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
413     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
414     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
415     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
416     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
417     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
418     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
419     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
420     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
421     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
422     "paddw           %%mm3, %%mm4       \n\t"\
423     "movq            %%mm2, %%mm0       \n\t"\
424     "movq            %%mm5, %%mm6       \n\t"\
425     "movq            %%mm4, %%mm3       \n\t"\
426     "punpcklwd       %%mm2, %%mm2       \n\t"\
427     "punpcklwd       %%mm5, %%mm5       \n\t"\
428     "punpcklwd       %%mm4, %%mm4       \n\t"\
429     "paddw           %%mm1, %%mm2       \n\t"\
430     "paddw           %%mm1, %%mm5       \n\t"\
431     "paddw           %%mm1, %%mm4       \n\t"\
432     "punpckhwd       %%mm0, %%mm0       \n\t"\
433     "punpckhwd       %%mm6, %%mm6       \n\t"\
434     "punpckhwd       %%mm3, %%mm3       \n\t"\
435     "paddw           %%mm7, %%mm0       \n\t"\
436     "paddw           %%mm7, %%mm6       \n\t"\
437     "paddw           %%mm7, %%mm3       \n\t"\
438     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
439     "packuswb        %%mm0, %%mm2       \n\t"\
440     "packuswb        %%mm6, %%mm5       \n\t"\
441     "packuswb        %%mm3, %%mm4       \n\t"\
442
443 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
444     "movq       "#b", "#q2"     \n\t" /* B */\
445     "movq       "#r", "#t"      \n\t" /* R */\
446     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
447     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
448     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
449     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
450     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
451     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
452     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
453     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
454     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
455     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
456 \
457     MOVNTQ(   q0,   (dst, index, 4))\
458     MOVNTQ(    b,  8(dst, index, 4))\
459     MOVNTQ(   q2, 16(dst, index, 4))\
460     MOVNTQ(   q3, 24(dst, index, 4))\
461 \
462     "add      $8, "#index"      \n\t"\
463     "cmp "#dstw", "#index"      \n\t"\
464     " jb      1b                \n\t"
465 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
466
467 static inline void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,
468                                           const int16_t **lumSrc, int lumFilterSize,
469                                           const int16_t *chrFilter, const int16_t **chrUSrc,
470                                           const int16_t **chrVSrc,
471                                           int chrFilterSize, const int16_t **alpSrc,
472                                           uint8_t *dest, long dstW, long dstY)
473 {
474     x86_reg dummy=0;
475     x86_reg dstW_reg = dstW;
476     x86_reg uv_off = c->uv_off << 1;
477
478     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
479         YSCALEYUV2PACKEDX_ACCURATE
480         YSCALEYUV2RGBX
481         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
482         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
483         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
484         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
485         "movq               "Y_TEMP"(%0), %%mm5         \n\t"
486         "psraw                        $3, %%mm1         \n\t"
487         "psraw                        $3, %%mm7         \n\t"
488         "packuswb                  %%mm7, %%mm1         \n\t"
489         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
490         YSCALEYUV2PACKEDX_END
491     } else {
492         YSCALEYUV2PACKEDX_ACCURATE
493         YSCALEYUV2RGBX
494         "pcmpeqd %%mm7, %%mm7 \n\t"
495         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
496         YSCALEYUV2PACKEDX_END
497     }
498 }
499
500 static inline void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,
501                                        const int16_t **lumSrc, int lumFilterSize,
502                                        const int16_t *chrFilter, const int16_t **chrUSrc,
503                                        const int16_t **chrVSrc,
504                                        int chrFilterSize, const int16_t **alpSrc,
505                                        uint8_t *dest, long dstW, long dstY)
506 {
507     x86_reg dummy=0;
508     x86_reg dstW_reg = dstW;
509     x86_reg uv_off = c->uv_off << 1;
510
511     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
512         YSCALEYUV2PACKEDX
513         YSCALEYUV2RGBX
514         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
515         "psraw                        $3, %%mm1         \n\t"
516         "psraw                        $3, %%mm7         \n\t"
517         "packuswb                  %%mm7, %%mm1         \n\t"
518         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
519         YSCALEYUV2PACKEDX_END
520     } else {
521         YSCALEYUV2PACKEDX
522         YSCALEYUV2RGBX
523         "pcmpeqd %%mm7, %%mm7 \n\t"
524         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
525         YSCALEYUV2PACKEDX_END
526     }
527 }
528
529 #define REAL_WRITERGB16(dst, dstw, index) \
530     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
531     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
532     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
533     "psrlq           $3, %%mm2  \n\t"\
534 \
535     "movq         %%mm2, %%mm1  \n\t"\
536     "movq         %%mm4, %%mm3  \n\t"\
537 \
538     "punpcklbw    %%mm7, %%mm3  \n\t"\
539     "punpcklbw    %%mm5, %%mm2  \n\t"\
540     "punpckhbw    %%mm7, %%mm4  \n\t"\
541     "punpckhbw    %%mm5, %%mm1  \n\t"\
542 \
543     "psllq           $3, %%mm3  \n\t"\
544     "psllq           $3, %%mm4  \n\t"\
545 \
546     "por          %%mm3, %%mm2  \n\t"\
547     "por          %%mm4, %%mm1  \n\t"\
548 \
549     MOVNTQ(%%mm2,  (dst, index, 2))\
550     MOVNTQ(%%mm1, 8(dst, index, 2))\
551 \
552     "add             $8, "#index"   \n\t"\
553     "cmp        "#dstw", "#index"   \n\t"\
554     " jb             1b             \n\t"
555 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
556
557 static inline void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,
558                                            const int16_t **lumSrc, int lumFilterSize,
559                                            const int16_t *chrFilter, const int16_t **chrUSrc,
560                                            const int16_t **chrVSrc,
561                                            int chrFilterSize, const int16_t **alpSrc,
562                                            uint8_t *dest, long dstW, long dstY)
563 {
564     x86_reg dummy=0;
565     x86_reg dstW_reg = dstW;
566     x86_reg uv_off = c->uv_off << 1;
567
568     YSCALEYUV2PACKEDX_ACCURATE
569     YSCALEYUV2RGBX
570     "pxor %%mm7, %%mm7 \n\t"
571     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
572 #ifdef DITHER1XBPP
573     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
574     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
575     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
576 #endif
577     WRITERGB16(%4, %5, %%REGa)
578     YSCALEYUV2PACKEDX_END
579 }
580
581 static inline void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,
582                                         const int16_t **lumSrc, int lumFilterSize,
583                                         const int16_t *chrFilter, const int16_t **chrUSrc,
584                                         const int16_t **chrVSrc,
585                                         int chrFilterSize, const int16_t **alpSrc,
586                                         uint8_t *dest, long dstW, long dstY)
587 {
588     x86_reg dummy=0;
589     x86_reg dstW_reg = dstW;
590     x86_reg uv_off = c->uv_off << 1;
591
592     YSCALEYUV2PACKEDX
593     YSCALEYUV2RGBX
594     "pxor %%mm7, %%mm7 \n\t"
595     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
596 #ifdef DITHER1XBPP
597     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
598     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
599     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
600 #endif
601     WRITERGB16(%4, %5, %%REGa)
602     YSCALEYUV2PACKEDX_END
603 }
604
605 #define REAL_WRITERGB15(dst, dstw, index) \
606     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
607     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
608     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
609     "psrlq           $3, %%mm2  \n\t"\
610     "psrlq           $1, %%mm5  \n\t"\
611 \
612     "movq         %%mm2, %%mm1  \n\t"\
613     "movq         %%mm4, %%mm3  \n\t"\
614 \
615     "punpcklbw    %%mm7, %%mm3  \n\t"\
616     "punpcklbw    %%mm5, %%mm2  \n\t"\
617     "punpckhbw    %%mm7, %%mm4  \n\t"\
618     "punpckhbw    %%mm5, %%mm1  \n\t"\
619 \
620     "psllq           $2, %%mm3  \n\t"\
621     "psllq           $2, %%mm4  \n\t"\
622 \
623     "por          %%mm3, %%mm2  \n\t"\
624     "por          %%mm4, %%mm1  \n\t"\
625 \
626     MOVNTQ(%%mm2,  (dst, index, 2))\
627     MOVNTQ(%%mm1, 8(dst, index, 2))\
628 \
629     "add             $8, "#index"   \n\t"\
630     "cmp        "#dstw", "#index"   \n\t"\
631     " jb             1b             \n\t"
632 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
633
634 static inline void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,
635                                            const int16_t **lumSrc, int lumFilterSize,
636                                            const int16_t *chrFilter, const int16_t **chrUSrc,
637                                            const int16_t **chrVSrc,
638                                            int chrFilterSize, const int16_t **alpSrc,
639                                            uint8_t *dest, long dstW, long dstY)
640 {
641     x86_reg dummy=0;
642     x86_reg dstW_reg = dstW;
643     x86_reg uv_off = c->uv_off << 1;
644
645     YSCALEYUV2PACKEDX_ACCURATE
646     YSCALEYUV2RGBX
647     "pxor %%mm7, %%mm7 \n\t"
648     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
649 #ifdef DITHER1XBPP
650     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
651     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
652     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
653 #endif
654     WRITERGB15(%4, %5, %%REGa)
655     YSCALEYUV2PACKEDX_END
656 }
657
658 static inline void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,
659                                         const int16_t **lumSrc, int lumFilterSize,
660                                         const int16_t *chrFilter, const int16_t **chrUSrc,
661                                         const int16_t **chrVSrc,
662                                         int chrFilterSize, const int16_t **alpSrc,
663                                         uint8_t *dest, long dstW, long dstY)
664 {
665     x86_reg dummy=0;
666     x86_reg dstW_reg = dstW;
667     x86_reg uv_off = c->uv_off << 1;
668
669     YSCALEYUV2PACKEDX
670     YSCALEYUV2RGBX
671     "pxor %%mm7, %%mm7 \n\t"
672     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
673 #ifdef DITHER1XBPP
674     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
675     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
676     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
677 #endif
678     WRITERGB15(%4, %5, %%REGa)
679     YSCALEYUV2PACKEDX_END
680 }
681
682 #define WRITEBGR24MMX(dst, dstw, index) \
683     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
684     "movq      %%mm2, %%mm1     \n\t" /* B */\
685     "movq      %%mm5, %%mm6     \n\t" /* R */\
686     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
687     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
688     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
689     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
690     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
691     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
692     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
693     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
694     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
695     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
696 \
697     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
698     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
699     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
700     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
701 \
702     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
703     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
704     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
705     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
706 \
707     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
708     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
709     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
710     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
711 \
712     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
713     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
714     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
715     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
716     MOVNTQ(%%mm0, (dst))\
717 \
718     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
719     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
720     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
721     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
722     MOVNTQ(%%mm6, 8(dst))\
723 \
724     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
725     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
726     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
727     MOVNTQ(%%mm5, 16(dst))\
728 \
729     "add         $24, "#dst"    \n\t"\
730 \
731     "add          $8, "#index"  \n\t"\
732     "cmp     "#dstw", "#index"  \n\t"\
733     " jb          1b            \n\t"
734
735 #define WRITEBGR24MMX2(dst, dstw, index) \
736     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
737     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
738     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
739     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
740     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
741     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
742 \
743     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
744     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
745     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
746 \
747     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
748     "por    %%mm1, %%mm6        \n\t"\
749     "por    %%mm3, %%mm6        \n\t"\
750     MOVNTQ(%%mm6, (dst))\
751 \
752     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
753     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
754     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
755     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
756 \
757     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
758     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
759     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
760 \
761     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
762     "por    %%mm3, %%mm6        \n\t"\
763     MOVNTQ(%%mm6, 8(dst))\
764 \
765     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
766     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
767     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
768 \
769     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
770     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
771     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
772 \
773     "por    %%mm1, %%mm3        \n\t"\
774     "por    %%mm3, %%mm6        \n\t"\
775     MOVNTQ(%%mm6, 16(dst))\
776 \
777     "add      $24, "#dst"       \n\t"\
778 \
779     "add       $8, "#index"     \n\t"\
780     "cmp  "#dstw", "#index"     \n\t"\
781     " jb       1b               \n\t"
782
783 #if COMPILE_TEMPLATE_MMX2
784 #undef WRITEBGR24
785 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
786 #else
787 #undef WRITEBGR24
788 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
789 #endif
790
791 static inline void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,
792                                           const int16_t **lumSrc, int lumFilterSize,
793                                           const int16_t *chrFilter, const int16_t **chrUSrc,
794                                           const int16_t **chrVSrc,
795                                           int chrFilterSize, const int16_t **alpSrc,
796                                           uint8_t *dest, long dstW, long dstY)
797 {
798     x86_reg dummy=0;
799     x86_reg dstW_reg = dstW;
800     x86_reg uv_off = c->uv_off << 1;
801
802     YSCALEYUV2PACKEDX_ACCURATE
803     YSCALEYUV2RGBX
804     "pxor %%mm7, %%mm7 \n\t"
805     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
806     "add %4, %%"REG_c"                        \n\t"
807     WRITEBGR24(%%REGc, %5, %%REGa)
808     :: "r" (&c->redDither),
809        "m" (dummy), "m" (dummy), "m" (dummy),
810        "r" (dest), "m" (dstW_reg), "m"(uv_off)
811     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
812     );
813 }
814
815 static inline void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,
816                                        const int16_t **lumSrc, int lumFilterSize,
817                                        const int16_t *chrFilter, const int16_t **chrUSrc,
818                                        const int16_t **chrVSrc,
819                                        int chrFilterSize, const int16_t **alpSrc,
820                                        uint8_t *dest, long dstW, long dstY)
821 {
822     x86_reg dummy=0;
823     x86_reg dstW_reg = dstW;
824     x86_reg uv_off = c->uv_off << 1;
825
826     YSCALEYUV2PACKEDX
827     YSCALEYUV2RGBX
828     "pxor                    %%mm7, %%mm7       \n\t"
829     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
830     "add                        %4, %%"REG_c"   \n\t"
831     WRITEBGR24(%%REGc, %5, %%REGa)
832     :: "r" (&c->redDither),
833        "m" (dummy), "m" (dummy), "m" (dummy),
834        "r" (dest),  "m" (dstW_reg), "m"(uv_off)
835     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
836     );
837 }
838
839 #define REAL_WRITEYUY2(dst, dstw, index) \
840     "packuswb  %%mm3, %%mm3     \n\t"\
841     "packuswb  %%mm4, %%mm4     \n\t"\
842     "packuswb  %%mm7, %%mm1     \n\t"\
843     "punpcklbw %%mm4, %%mm3     \n\t"\
844     "movq      %%mm1, %%mm7     \n\t"\
845     "punpcklbw %%mm3, %%mm1     \n\t"\
846     "punpckhbw %%mm3, %%mm7     \n\t"\
847 \
848     MOVNTQ(%%mm1, (dst, index, 2))\
849     MOVNTQ(%%mm7, 8(dst, index, 2))\
850 \
851     "add          $8, "#index"  \n\t"\
852     "cmp     "#dstw", "#index"  \n\t"\
853     " jb          1b            \n\t"
854 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
855
856 static inline void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,
857                                             const int16_t **lumSrc, int lumFilterSize,
858                                             const int16_t *chrFilter, const int16_t **chrUSrc,
859                                             const int16_t **chrVSrc,
860                                             int chrFilterSize, const int16_t **alpSrc,
861                                             uint8_t *dest, long dstW, long dstY)
862 {
863     x86_reg dummy=0;
864     x86_reg dstW_reg = dstW;
865     x86_reg uv_off = c->uv_off << 1;
866
867     YSCALEYUV2PACKEDX_ACCURATE
868     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
869     "psraw $3, %%mm3    \n\t"
870     "psraw $3, %%mm4    \n\t"
871     "psraw $3, %%mm1    \n\t"
872     "psraw $3, %%mm7    \n\t"
873     WRITEYUY2(%4, %5, %%REGa)
874     YSCALEYUV2PACKEDX_END
875 }
876
877 static inline void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,
878                                          const int16_t **lumSrc, int lumFilterSize,
879                                          const int16_t *chrFilter, const int16_t **chrUSrc,
880                                          const int16_t **chrVSrc,
881                                          int chrFilterSize, const int16_t **alpSrc,
882                                          uint8_t *dest, long dstW, long dstY)
883 {
884     x86_reg dummy=0;
885     x86_reg dstW_reg = dstW;
886     x86_reg uv_off = c->uv_off << 1;
887
888     YSCALEYUV2PACKEDX
889     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
890     "psraw $3, %%mm3    \n\t"
891     "psraw $3, %%mm4    \n\t"
892     "psraw $3, %%mm1    \n\t"
893     "psraw $3, %%mm7    \n\t"
894     WRITEYUY2(%4, %5, %%REGa)
895     YSCALEYUV2PACKEDX_END
896 }
897
898 #define REAL_YSCALEYUV2RGB_UV(index, c, uv_off) \
899     "xor            "#index", "#index"  \n\t"\
900     ".p2align              4            \n\t"\
901     "1:                                 \n\t"\
902     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
903     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
904     "add           "#uv_off", "#index"  \n\t" \
905     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
906     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
907     "sub           "#uv_off", "#index"  \n\t" \
908     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
909     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
910     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
911     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
912     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
913     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
914     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
915     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
916     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
917     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
918     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
919     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
920     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
921     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
922     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
923     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
924
925 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
926     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
927     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
928     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
929     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
930     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
931     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
932     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
933     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
934     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
935     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
936     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
937     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
938
939 #define REAL_YSCALEYUV2RGB_COEFF(c) \
940     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
941     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
942     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
943     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
944     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
945     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
946     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
947     "paddw             %%mm3, %%mm4     \n\t"\
948     "movq              %%mm2, %%mm0     \n\t"\
949     "movq              %%mm5, %%mm6     \n\t"\
950     "movq              %%mm4, %%mm3     \n\t"\
951     "punpcklwd         %%mm2, %%mm2     \n\t"\
952     "punpcklwd         %%mm5, %%mm5     \n\t"\
953     "punpcklwd         %%mm4, %%mm4     \n\t"\
954     "paddw             %%mm1, %%mm2     \n\t"\
955     "paddw             %%mm1, %%mm5     \n\t"\
956     "paddw             %%mm1, %%mm4     \n\t"\
957     "punpckhwd         %%mm0, %%mm0     \n\t"\
958     "punpckhwd         %%mm6, %%mm6     \n\t"\
959     "punpckhwd         %%mm3, %%mm3     \n\t"\
960     "paddw             %%mm7, %%mm0     \n\t"\
961     "paddw             %%mm7, %%mm6     \n\t"\
962     "paddw             %%mm7, %%mm3     \n\t"\
963     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
964     "packuswb          %%mm0, %%mm2     \n\t"\
965     "packuswb          %%mm6, %%mm5     \n\t"\
966     "packuswb          %%mm3, %%mm4     \n\t"\
967
968 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
969
970 #define YSCALEYUV2RGB(index, c, uv_off) \
971     REAL_YSCALEYUV2RGB_UV(index, c, uv_off) \
972     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
973     REAL_YSCALEYUV2RGB_COEFF(c)
974
975 /**
976  * vertical bilinear scale YV12 to RGB
977  */
978 static inline void RENAME(yuv2rgb32_2)(SwsContext *c, const uint16_t *buf0,
979                                        const uint16_t *buf1, const uint16_t *ubuf0,
980                                        const uint16_t *ubuf1, const uint16_t *vbuf0,
981                                        const uint16_t *vbuf1, const uint16_t *abuf0,
982                                        const uint16_t *abuf1, uint8_t *dest,
983                                        int dstW, int yalpha, int uvalpha, int y)
984 {
985     x86_reg uv_off = c->uv_off << 1;
986
987     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
988 #if ARCH_X86_64
989         __asm__ volatile(
990             YSCALEYUV2RGB(%%r8, %5, %8)
991             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
992             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
993             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
994             "packuswb            %%mm7, %%mm1       \n\t"
995             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
996             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "r" (dest),
997                "a" (&c->redDither),
998                "r" (abuf0), "r" (abuf1), "m"(uv_off)
999             : "%r8"
1000         );
1001 #else
1002         *(const uint16_t **)(&c->u_temp)=abuf0;
1003         *(const uint16_t **)(&c->v_temp)=abuf1;
1004         __asm__ volatile(
1005             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1006             "mov        %4, %%"REG_b"               \n\t"
1007             "push %%"REG_BP"                        \n\t"
1008             YSCALEYUV2RGB(%%REGBP, %5, %6)
1009             "push                   %0              \n\t"
1010             "push                   %1              \n\t"
1011             "mov          "U_TEMP"(%5), %0          \n\t"
1012             "mov          "V_TEMP"(%5), %1          \n\t"
1013             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
1014             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1015             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1016             "packuswb            %%mm7, %%mm1       \n\t"
1017             "pop                    %1              \n\t"
1018             "pop                    %0              \n\t"
1019             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1020             "pop %%"REG_BP"                         \n\t"
1021             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1022             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1023                "a" (&c->redDither), "m"(uv_off)
1024         );
1025 #endif
1026     } else {
1027         __asm__ volatile(
1028             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1029             "mov        %4, %%"REG_b"               \n\t"
1030             "push %%"REG_BP"                        \n\t"
1031             YSCALEYUV2RGB(%%REGBP, %5, %6)
1032             "pcmpeqd %%mm7, %%mm7                   \n\t"
1033             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1034             "pop %%"REG_BP"                         \n\t"
1035             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1036             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1037                "a" (&c->redDither), "m"(uv_off)
1038         );
1039     }
1040 }
1041
1042 static inline void RENAME(yuv2bgr24_2)(SwsContext *c, const uint16_t *buf0,
1043                                        const uint16_t *buf1, const uint16_t *ubuf0,
1044                                        const uint16_t *ubuf1, const uint16_t *vbuf0,
1045                                        const uint16_t *vbuf1, const uint16_t *abuf0,
1046                                        const uint16_t *abuf1, uint8_t *dest,
1047                                        int dstW, int yalpha, int uvalpha, int y)
1048 {
1049     x86_reg uv_off = c->uv_off << 1;
1050
1051     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1052     __asm__ volatile(
1053         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1054         "mov        %4, %%"REG_b"               \n\t"
1055         "push %%"REG_BP"                        \n\t"
1056         YSCALEYUV2RGB(%%REGBP, %5, %6)
1057         "pxor    %%mm7, %%mm7                   \n\t"
1058         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1059         "pop %%"REG_BP"                         \n\t"
1060         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1061         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1062            "a" (&c->redDither), "m"(uv_off)
1063     );
1064 }
1065
1066 static inline void RENAME(yuv2rgb555_2)(SwsContext *c, const uint16_t *buf0,
1067                                         const uint16_t *buf1, const uint16_t *ubuf0,
1068                                         const uint16_t *ubuf1, const uint16_t *vbuf0,
1069                                         const uint16_t *vbuf1, const uint16_t *abuf0,
1070                                         const uint16_t *abuf1, uint8_t *dest,
1071                                         int dstW, int yalpha, int uvalpha, int y)
1072 {
1073     x86_reg uv_off = c->uv_off << 1;
1074
1075     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1076     __asm__ volatile(
1077         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1078         "mov        %4, %%"REG_b"               \n\t"
1079         "push %%"REG_BP"                        \n\t"
1080         YSCALEYUV2RGB(%%REGBP, %5, %6)
1081         "pxor    %%mm7, %%mm7                   \n\t"
1082         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1083 #ifdef DITHER1XBPP
1084         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1085         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1086         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1087 #endif
1088         WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1089         "pop %%"REG_BP"                         \n\t"
1090         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1091         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1092            "a" (&c->redDither), "m"(uv_off)
1093     );
1094 }
1095
1096 static inline void RENAME(yuv2rgb565_2)(SwsContext *c, const uint16_t *buf0,
1097                                         const uint16_t *buf1, const uint16_t *ubuf0,
1098                                         const uint16_t *ubuf1, const uint16_t *vbuf0,
1099                                         const uint16_t *vbuf1, const uint16_t *abuf0,
1100                                         const uint16_t *abuf1, uint8_t *dest,
1101                                         int dstW, int yalpha, int uvalpha, int y)
1102 {
1103     x86_reg uv_off = c->uv_off << 1;
1104
1105     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1106     __asm__ volatile(
1107         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1108         "mov        %4, %%"REG_b"               \n\t"
1109         "push %%"REG_BP"                        \n\t"
1110         YSCALEYUV2RGB(%%REGBP, %5, %6)
1111         "pxor    %%mm7, %%mm7                   \n\t"
1112         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1113 #ifdef DITHER1XBPP
1114         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1115         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1116         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1117 #endif
1118         WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1119         "pop %%"REG_BP"                         \n\t"
1120         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1121         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1122            "a" (&c->redDither), "m"(uv_off)
1123     );
1124 }
1125
1126 #define REAL_YSCALEYUV2PACKED(index, c, uv_off) \
1127     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
1128     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
1129     "psraw                $3, %%mm0                           \n\t"\
1130     "psraw                $3, %%mm1                           \n\t"\
1131     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1132     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1133     "xor            "#index", "#index"                        \n\t"\
1134     ".p2align              4            \n\t"\
1135     "1:                                 \n\t"\
1136     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1137     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1138     "add           "#uv_off", "#index"  \n\t" \
1139     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1140     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1141     "sub           "#uv_off", "#index"  \n\t" \
1142     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
1143     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
1144     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
1145     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
1146     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
1147     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1148     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1149     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
1150     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
1151     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
1152     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
1153     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
1154     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
1155     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
1156     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
1157     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1158     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1159     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1160     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1161     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1162     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1163
1164 #define YSCALEYUV2PACKED(index, c, uv_off)  REAL_YSCALEYUV2PACKED(index, c, uv_off)
1165
1166 static inline void RENAME(yuv2yuyv422_2)(SwsContext *c, const uint16_t *buf0,
1167                                          const uint16_t *buf1, const uint16_t *ubuf0,
1168                                          const uint16_t *ubuf1, const uint16_t *vbuf0,
1169                                          const uint16_t *vbuf1, const uint16_t *abuf0,
1170                                          const uint16_t *abuf1, uint8_t *dest,
1171                                          int dstW, int yalpha, int uvalpha, int y)
1172 {
1173     x86_reg uv_off = c->uv_off << 1;
1174
1175     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1176     __asm__ volatile(
1177         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1178         "mov %4, %%"REG_b"                        \n\t"
1179         "push %%"REG_BP"                        \n\t"
1180         YSCALEYUV2PACKED(%%REGBP, %5, %6)
1181         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1182         "pop %%"REG_BP"                         \n\t"
1183         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1184         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1185            "a" (&c->redDither), "m"(uv_off)
1186     );
1187 }
1188
1189 #define REAL_YSCALEYUV2RGB1(index, c, uv_off) \
1190     "xor            "#index", "#index"  \n\t"\
1191     ".p2align              4            \n\t"\
1192     "1:                                 \n\t"\
1193     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1194     "add           "#uv_off", "#index"  \n\t" \
1195     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1196     "sub           "#uv_off", "#index"  \n\t" \
1197     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1198     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1199     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1200     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1201     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1202     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1203     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1204     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1205     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1206     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1207     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1208     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1209     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1210     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1211     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1212     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1213     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1214     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1215     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1216     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1217     "paddw             %%mm3, %%mm4     \n\t"\
1218     "movq              %%mm2, %%mm0     \n\t"\
1219     "movq              %%mm5, %%mm6     \n\t"\
1220     "movq              %%mm4, %%mm3     \n\t"\
1221     "punpcklwd         %%mm2, %%mm2     \n\t"\
1222     "punpcklwd         %%mm5, %%mm5     \n\t"\
1223     "punpcklwd         %%mm4, %%mm4     \n\t"\
1224     "paddw             %%mm1, %%mm2     \n\t"\
1225     "paddw             %%mm1, %%mm5     \n\t"\
1226     "paddw             %%mm1, %%mm4     \n\t"\
1227     "punpckhwd         %%mm0, %%mm0     \n\t"\
1228     "punpckhwd         %%mm6, %%mm6     \n\t"\
1229     "punpckhwd         %%mm3, %%mm3     \n\t"\
1230     "paddw             %%mm7, %%mm0     \n\t"\
1231     "paddw             %%mm7, %%mm6     \n\t"\
1232     "paddw             %%mm7, %%mm3     \n\t"\
1233     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1234     "packuswb          %%mm0, %%mm2     \n\t"\
1235     "packuswb          %%mm6, %%mm5     \n\t"\
1236     "packuswb          %%mm3, %%mm4     \n\t"\
1237
1238 #define YSCALEYUV2RGB1(index, c, uv_off)  REAL_YSCALEYUV2RGB1(index, c, uv_off)
1239
1240 // do vertical chrominance interpolation
1241 #define REAL_YSCALEYUV2RGB1b(index, c, uv_off) \
1242     "xor            "#index", "#index"  \n\t"\
1243     ".p2align              4            \n\t"\
1244     "1:                                 \n\t"\
1245     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1246     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1247     "add           "#uv_off", "#index"  \n\t" \
1248     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1249     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1250     "sub           "#uv_off", "#index"  \n\t" \
1251     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1252     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1253     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
1254     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
1255     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1256     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1257     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1258     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1259     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1260     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1261     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1262     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1263     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1264     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1265     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1266     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1267     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1268     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1269     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1270     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1271     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1272     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1273     "paddw             %%mm3, %%mm4     \n\t"\
1274     "movq              %%mm2, %%mm0     \n\t"\
1275     "movq              %%mm5, %%mm6     \n\t"\
1276     "movq              %%mm4, %%mm3     \n\t"\
1277     "punpcklwd         %%mm2, %%mm2     \n\t"\
1278     "punpcklwd         %%mm5, %%mm5     \n\t"\
1279     "punpcklwd         %%mm4, %%mm4     \n\t"\
1280     "paddw             %%mm1, %%mm2     \n\t"\
1281     "paddw             %%mm1, %%mm5     \n\t"\
1282     "paddw             %%mm1, %%mm4     \n\t"\
1283     "punpckhwd         %%mm0, %%mm0     \n\t"\
1284     "punpckhwd         %%mm6, %%mm6     \n\t"\
1285     "punpckhwd         %%mm3, %%mm3     \n\t"\
1286     "paddw             %%mm7, %%mm0     \n\t"\
1287     "paddw             %%mm7, %%mm6     \n\t"\
1288     "paddw             %%mm7, %%mm3     \n\t"\
1289     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1290     "packuswb          %%mm0, %%mm2     \n\t"\
1291     "packuswb          %%mm6, %%mm5     \n\t"\
1292     "packuswb          %%mm3, %%mm4     \n\t"\
1293
1294 #define YSCALEYUV2RGB1b(index, c, uv_off)  REAL_YSCALEYUV2RGB1b(index, c, uv_off)
1295
1296 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
1297     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
1298     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
1299     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
1300     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
1301     "packuswb          %%mm1, %%mm7     \n\t"
1302 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
1303
1304 /**
1305  * YV12 to RGB without scaling or interpolating
1306  */
1307 static inline void RENAME(yuv2rgb32_1)(SwsContext *c, const uint16_t *buf0,
1308                                        const uint16_t *ubuf0, const uint16_t *ubuf1,
1309                                        const uint16_t *vbuf0, const uint16_t *vbuf1,
1310                                        const uint16_t *abuf0, uint8_t *dest,
1311                                        int dstW, int uvalpha, enum PixelFormat dstFormat,
1312                                        int flags, int y)
1313 {
1314     x86_reg uv_off = c->uv_off << 1;
1315     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1316
1317     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1318         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1319             __asm__ volatile(
1320                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1321                 "mov        %4, %%"REG_b"               \n\t"
1322                 "push %%"REG_BP"                        \n\t"
1323                 YSCALEYUV2RGB1(%%REGBP, %5, %6)
1324                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1325                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1326                 "pop %%"REG_BP"                         \n\t"
1327                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1328                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1329                    "a" (&c->redDither), "m"(uv_off)
1330             );
1331         } else {
1332             __asm__ volatile(
1333                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1334                 "mov        %4, %%"REG_b"               \n\t"
1335                 "push %%"REG_BP"                        \n\t"
1336                 YSCALEYUV2RGB1(%%REGBP, %5, %6)
1337                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1338                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1339                 "pop %%"REG_BP"                         \n\t"
1340                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1341                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1342                    "a" (&c->redDither), "m"(uv_off)
1343             );
1344         }
1345     } else {
1346         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1347             __asm__ volatile(
1348                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1349                 "mov        %4, %%"REG_b"               \n\t"
1350                 "push %%"REG_BP"                        \n\t"
1351                 YSCALEYUV2RGB1b(%%REGBP, %5, %6)
1352                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1353                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1354                 "pop %%"REG_BP"                         \n\t"
1355                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1356                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1357                    "a" (&c->redDither), "m"(uv_off)
1358             );
1359         } else {
1360             __asm__ volatile(
1361                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1362                 "mov        %4, %%"REG_b"               \n\t"
1363                 "push %%"REG_BP"                        \n\t"
1364                 YSCALEYUV2RGB1b(%%REGBP, %5, %6)
1365                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1366                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1367                 "pop %%"REG_BP"                         \n\t"
1368                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1369                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1370                    "a" (&c->redDither), "m"(uv_off)
1371             );
1372         }
1373     }
1374 }
1375
1376 static inline void RENAME(yuv2bgr24_1)(SwsContext *c, const uint16_t *buf0,
1377                                        const uint16_t *ubuf0, const uint16_t *ubuf1,
1378                                        const uint16_t *vbuf0, const uint16_t *vbuf1,
1379                                        const uint16_t *abuf0, uint8_t *dest,
1380                                        int dstW, int uvalpha, enum PixelFormat dstFormat,
1381                                        int flags, int y)
1382 {
1383     x86_reg uv_off = c->uv_off << 1;
1384     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1385
1386     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1387         __asm__ volatile(
1388             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1389             "mov        %4, %%"REG_b"               \n\t"
1390             "push %%"REG_BP"                        \n\t"
1391             YSCALEYUV2RGB1(%%REGBP, %5, %6)
1392             "pxor    %%mm7, %%mm7                   \n\t"
1393             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1394             "pop %%"REG_BP"                         \n\t"
1395             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1396             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1397                "a" (&c->redDither), "m"(uv_off)
1398         );
1399     } else {
1400         __asm__ volatile(
1401             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1402             "mov        %4, %%"REG_b"               \n\t"
1403             "push %%"REG_BP"                        \n\t"
1404             YSCALEYUV2RGB1b(%%REGBP, %5, %6)
1405             "pxor    %%mm7, %%mm7                   \n\t"
1406             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1407             "pop %%"REG_BP"                         \n\t"
1408             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1409             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1410                "a" (&c->redDither), "m"(uv_off)
1411         );
1412     }
1413 }
1414
1415 static inline void RENAME(yuv2rgb555_1)(SwsContext *c, const uint16_t *buf0,
1416                                         const uint16_t *ubuf0, const uint16_t *ubuf1,
1417                                         const uint16_t *vbuf0, const uint16_t *vbuf1,
1418                                         const uint16_t *abuf0, uint8_t *dest,
1419                                         int dstW, int uvalpha, enum PixelFormat dstFormat,
1420                                         int flags, int y)
1421 {
1422     x86_reg uv_off = c->uv_off << 1;
1423     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1424
1425     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1426         __asm__ volatile(
1427             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1428             "mov        %4, %%"REG_b"               \n\t"
1429             "push %%"REG_BP"                        \n\t"
1430             YSCALEYUV2RGB1(%%REGBP, %5, %6)
1431             "pxor    %%mm7, %%mm7                   \n\t"
1432             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1433 #ifdef DITHER1XBPP
1434             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1435             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1436             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1437 #endif
1438             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1439             "pop %%"REG_BP"                         \n\t"
1440             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1441             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1442                "a" (&c->redDither), "m"(uv_off)
1443         );
1444     } else {
1445         __asm__ volatile(
1446             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1447             "mov        %4, %%"REG_b"               \n\t"
1448             "push %%"REG_BP"                        \n\t"
1449             YSCALEYUV2RGB1b(%%REGBP, %5, %6)
1450             "pxor    %%mm7, %%mm7                   \n\t"
1451             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1452 #ifdef DITHER1XBPP
1453             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1454             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1455             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1456 #endif
1457             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1458             "pop %%"REG_BP"                         \n\t"
1459             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1460             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1461                "a" (&c->redDither), "m"(uv_off)
1462         );
1463     }
1464 }
1465
1466 static inline void RENAME(yuv2rgb565_1)(SwsContext *c, const uint16_t *buf0,
1467                                         const uint16_t *ubuf0, const uint16_t *ubuf1,
1468                                         const uint16_t *vbuf0, const uint16_t *vbuf1,
1469                                         const uint16_t *abuf0, uint8_t *dest,
1470                                         int dstW, int uvalpha, enum PixelFormat dstFormat,
1471                                         int flags, int y)
1472 {
1473     x86_reg uv_off = c->uv_off << 1;
1474     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1475
1476     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1477         __asm__ volatile(
1478             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1479             "mov        %4, %%"REG_b"               \n\t"
1480             "push %%"REG_BP"                        \n\t"
1481             YSCALEYUV2RGB1(%%REGBP, %5, %6)
1482             "pxor    %%mm7, %%mm7                   \n\t"
1483             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1484 #ifdef DITHER1XBPP
1485             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1486             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1487             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1488 #endif
1489             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1490             "pop %%"REG_BP"                         \n\t"
1491             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1492             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1493                "a" (&c->redDither), "m"(uv_off)
1494         );
1495     } else {
1496         __asm__ volatile(
1497             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1498             "mov        %4, %%"REG_b"               \n\t"
1499             "push %%"REG_BP"                        \n\t"
1500             YSCALEYUV2RGB1b(%%REGBP, %5, %6)
1501             "pxor    %%mm7, %%mm7                   \n\t"
1502             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1503 #ifdef DITHER1XBPP
1504             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1505             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1506             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1507 #endif
1508             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1509             "pop %%"REG_BP"                         \n\t"
1510             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1511             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1512                "a" (&c->redDither), "m"(uv_off)
1513         );
1514     }
1515 }
1516
1517 #define REAL_YSCALEYUV2PACKED1(index, c, uv_off) \
1518     "xor            "#index", "#index"  \n\t"\
1519     ".p2align              4            \n\t"\
1520     "1:                                 \n\t"\
1521     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1522     "add           "#uv_off", "#index"  \n\t" \
1523     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1524     "sub           "#uv_off", "#index"  \n\t" \
1525     "psraw                $7, %%mm3     \n\t" \
1526     "psraw                $7, %%mm4     \n\t" \
1527     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1528     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1529     "psraw                $7, %%mm1     \n\t" \
1530     "psraw                $7, %%mm7     \n\t" \
1531
1532 #define YSCALEYUV2PACKED1(index, c, uv_off)  REAL_YSCALEYUV2PACKED1(index, c, uv_off)
1533
1534 #define REAL_YSCALEYUV2PACKED1b(index, c, uv_off) \
1535     "xor "#index", "#index"             \n\t"\
1536     ".p2align              4            \n\t"\
1537     "1:                                 \n\t"\
1538     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1539     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1540     "add           "#uv_off", "#index"  \n\t" \
1541     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1542     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1543     "sub           "#uv_off", "#index"  \n\t" \
1544     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1545     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1546     "psrlw                $8, %%mm3     \n\t" \
1547     "psrlw                $8, %%mm4     \n\t" \
1548     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1549     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1550     "psraw                $7, %%mm1     \n\t" \
1551     "psraw                $7, %%mm7     \n\t"
1552 #define YSCALEYUV2PACKED1b(index, c, uv_off)  REAL_YSCALEYUV2PACKED1b(index, c, uv_off)
1553
1554 static inline void RENAME(yuv2yuyv422_1)(SwsContext *c, const uint16_t *buf0,
1555                                          const uint16_t *ubuf0, const uint16_t *ubuf1,
1556                                          const uint16_t *vbuf0, const uint16_t *vbuf1,
1557                                          const uint16_t *abuf0, uint8_t *dest,
1558                                          int dstW, int uvalpha, enum PixelFormat dstFormat,
1559                                          int flags, int y)
1560 {
1561     x86_reg uv_off = c->uv_off << 1;
1562     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1563
1564     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1565         __asm__ volatile(
1566             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1567             "mov        %4, %%"REG_b"               \n\t"
1568             "push %%"REG_BP"                        \n\t"
1569             YSCALEYUV2PACKED1(%%REGBP, %5, %6)
1570             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1571             "pop %%"REG_BP"                         \n\t"
1572             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1573             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1574                "a" (&c->redDither), "m"(uv_off)
1575         );
1576     } else {
1577         __asm__ volatile(
1578             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1579             "mov        %4, %%"REG_b"               \n\t"
1580             "push %%"REG_BP"                        \n\t"
1581             YSCALEYUV2PACKED1b(%%REGBP, %5, %6)
1582             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1583             "pop %%"REG_BP"                         \n\t"
1584             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1585             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1586                "a" (&c->redDither), "m"(uv_off)
1587         );
1588     }
1589 }
1590
1591 #if !COMPILE_TEMPLATE_MMX2
1592 //FIXME yuy2* can read up to 7 samples too much
1593
1594 static inline void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1595 {
1596     __asm__ volatile(
1597         "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1598         "mov                    %0, %%"REG_a"       \n\t"
1599         "1:                                         \n\t"
1600         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1601         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1602         "pand                %%mm2, %%mm0           \n\t"
1603         "pand                %%mm2, %%mm1           \n\t"
1604         "packuswb            %%mm1, %%mm0           \n\t"
1605         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1606         "add                    $8, %%"REG_a"       \n\t"
1607         " js                    1b                  \n\t"
1608         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1609         : "%"REG_a
1610     );
1611 }
1612
1613 static inline void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1614 {
1615     __asm__ volatile(
1616         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1617         "mov                    %0, %%"REG_a"       \n\t"
1618         "1:                                         \n\t"
1619         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1620         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1621         "psrlw                  $8, %%mm0           \n\t"
1622         "psrlw                  $8, %%mm1           \n\t"
1623         "packuswb            %%mm1, %%mm0           \n\t"
1624         "movq                %%mm0, %%mm1           \n\t"
1625         "psrlw                  $8, %%mm0           \n\t"
1626         "pand                %%mm4, %%mm1           \n\t"
1627         "packuswb            %%mm0, %%mm0           \n\t"
1628         "packuswb            %%mm1, %%mm1           \n\t"
1629         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1630         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1631         "add                    $4, %%"REG_a"       \n\t"
1632         " js                    1b                  \n\t"
1633         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1634         : "%"REG_a
1635     );
1636     assert(src1 == src2);
1637 }
1638
1639 static inline void RENAME(LEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1640 {
1641     __asm__ volatile(
1642         "mov                    %0, %%"REG_a"       \n\t"
1643         "1:                                         \n\t"
1644         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1645         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1646         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1647         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1648         "psrlw                  $8, %%mm0           \n\t"
1649         "psrlw                  $8, %%mm1           \n\t"
1650         "psrlw                  $8, %%mm2           \n\t"
1651         "psrlw                  $8, %%mm3           \n\t"
1652         "packuswb            %%mm1, %%mm0           \n\t"
1653         "packuswb            %%mm3, %%mm2           \n\t"
1654         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1655         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1656         "add                    $8, %%"REG_a"       \n\t"
1657         " js                    1b                  \n\t"
1658         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1659         : "%"REG_a
1660     );
1661 }
1662
1663 /* This is almost identical to the previous, end exists only because
1664  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1665 static inline void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1666 {
1667     __asm__ volatile(
1668         "mov                  %0, %%"REG_a"         \n\t"
1669         "1:                                         \n\t"
1670         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1671         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1672         "psrlw                $8, %%mm0             \n\t"
1673         "psrlw                $8, %%mm1             \n\t"
1674         "packuswb          %%mm1, %%mm0             \n\t"
1675         "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1676         "add                  $8, %%"REG_a"         \n\t"
1677         " js                  1b                    \n\t"
1678         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1679         : "%"REG_a
1680     );
1681 }
1682
1683 static inline void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1684 {
1685     __asm__ volatile(
1686         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1687         "mov                    %0, %%"REG_a"       \n\t"
1688         "1:                                         \n\t"
1689         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1690         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1691         "pand                %%mm4, %%mm0           \n\t"
1692         "pand                %%mm4, %%mm1           \n\t"
1693         "packuswb            %%mm1, %%mm0           \n\t"
1694         "movq                %%mm0, %%mm1           \n\t"
1695         "psrlw                  $8, %%mm0           \n\t"
1696         "pand                %%mm4, %%mm1           \n\t"
1697         "packuswb            %%mm0, %%mm0           \n\t"
1698         "packuswb            %%mm1, %%mm1           \n\t"
1699         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1700         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1701         "add                    $4, %%"REG_a"       \n\t"
1702         " js                    1b                  \n\t"
1703         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1704         : "%"REG_a
1705     );
1706     assert(src1 == src2);
1707 }
1708
1709 static inline void RENAME(BEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1710 {
1711     __asm__ volatile(
1712         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1713         "mov                    %0, %%"REG_a"       \n\t"
1714         "1:                                         \n\t"
1715         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1716         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1717         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1718         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1719         "pand                %%mm4, %%mm0           \n\t"
1720         "pand                %%mm4, %%mm1           \n\t"
1721         "pand                %%mm4, %%mm2           \n\t"
1722         "pand                %%mm4, %%mm3           \n\t"
1723         "packuswb            %%mm1, %%mm0           \n\t"
1724         "packuswb            %%mm3, %%mm2           \n\t"
1725         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1726         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1727         "add                    $8, %%"REG_a"       \n\t"
1728         " js                    1b                  \n\t"
1729         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1730         : "%"REG_a
1731     );
1732 }
1733
1734 static inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1735                                     const uint8_t *src, long width)
1736 {
1737     __asm__ volatile(
1738         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1739         "mov                    %0, %%"REG_a"       \n\t"
1740         "1:                                         \n\t"
1741         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1742         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1743         "movq                %%mm0, %%mm2           \n\t"
1744         "movq                %%mm1, %%mm3           \n\t"
1745         "pand                %%mm4, %%mm0           \n\t"
1746         "pand                %%mm4, %%mm1           \n\t"
1747         "psrlw                  $8, %%mm2           \n\t"
1748         "psrlw                  $8, %%mm3           \n\t"
1749         "packuswb            %%mm1, %%mm0           \n\t"
1750         "packuswb            %%mm3, %%mm2           \n\t"
1751         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1752         "movq                %%mm2, (%3, %%"REG_a") \n\t"
1753         "add                    $8, %%"REG_a"       \n\t"
1754         " js                    1b                  \n\t"
1755         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1756         : "%"REG_a
1757     );
1758 }
1759
1760 static inline void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1761                                     const uint8_t *src1, const uint8_t *src2,
1762                                     long width, uint32_t *unused)
1763 {
1764     RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1765 }
1766
1767 static inline void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1768                                     const uint8_t *src1, const uint8_t *src2,
1769                                     long width, uint32_t *unused)
1770 {
1771     RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1772 }
1773 #endif /* !COMPILE_TEMPLATE_MMX2 */
1774
1775 static inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, const uint8_t *src, long width, enum PixelFormat srcFormat)
1776 {
1777
1778     if(srcFormat == PIX_FMT_BGR24) {
1779         __asm__ volatile(
1780             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1781             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1782             :
1783         );
1784     } else {
1785         __asm__ volatile(
1786             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1787             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1788             :
1789         );
1790     }
1791
1792     __asm__ volatile(
1793         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1794         "mov                        %2, %%"REG_a"   \n\t"
1795         "pxor                    %%mm7, %%mm7       \n\t"
1796         "1:                                         \n\t"
1797         PREFETCH"               64(%0)              \n\t"
1798         "movd                     (%0), %%mm0       \n\t"
1799         "movd                    2(%0), %%mm1       \n\t"
1800         "movd                    6(%0), %%mm2       \n\t"
1801         "movd                    8(%0), %%mm3       \n\t"
1802         "add                       $12, %0          \n\t"
1803         "punpcklbw               %%mm7, %%mm0       \n\t"
1804         "punpcklbw               %%mm7, %%mm1       \n\t"
1805         "punpcklbw               %%mm7, %%mm2       \n\t"
1806         "punpcklbw               %%mm7, %%mm3       \n\t"
1807         "pmaddwd                 %%mm5, %%mm0       \n\t"
1808         "pmaddwd                 %%mm6, %%mm1       \n\t"
1809         "pmaddwd                 %%mm5, %%mm2       \n\t"
1810         "pmaddwd                 %%mm6, %%mm3       \n\t"
1811         "paddd                   %%mm1, %%mm0       \n\t"
1812         "paddd                   %%mm3, %%mm2       \n\t"
1813         "paddd                   %%mm4, %%mm0       \n\t"
1814         "paddd                   %%mm4, %%mm2       \n\t"
1815         "psrad                     $15, %%mm0       \n\t"
1816         "psrad                     $15, %%mm2       \n\t"
1817         "packssdw                %%mm2, %%mm0       \n\t"
1818         "packuswb                %%mm0, %%mm0       \n\t"
1819         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1820         "add                        $4, %%"REG_a"   \n\t"
1821         " js                        1b              \n\t"
1822     : "+r" (src)
1823     : "r" (dst+width), "g" ((x86_reg)-width)
1824     : "%"REG_a
1825     );
1826 }
1827
1828 static inline void RENAME(bgr24ToUV_mmx)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src, long width, enum PixelFormat srcFormat)
1829 {
1830     __asm__ volatile(
1831         "movq                    24(%4), %%mm6       \n\t"
1832         "mov                        %3, %%"REG_a"   \n\t"
1833         "pxor                    %%mm7, %%mm7       \n\t"
1834         "1:                                         \n\t"
1835         PREFETCH"               64(%0)              \n\t"
1836         "movd                     (%0), %%mm0       \n\t"
1837         "movd                    2(%0), %%mm1       \n\t"
1838         "punpcklbw               %%mm7, %%mm0       \n\t"
1839         "punpcklbw               %%mm7, %%mm1       \n\t"
1840         "movq                    %%mm0, %%mm2       \n\t"
1841         "movq                    %%mm1, %%mm3       \n\t"
1842         "pmaddwd                  (%4), %%mm0       \n\t"
1843         "pmaddwd                 8(%4), %%mm1       \n\t"
1844         "pmaddwd                16(%4), %%mm2       \n\t"
1845         "pmaddwd                 %%mm6, %%mm3       \n\t"
1846         "paddd                   %%mm1, %%mm0       \n\t"
1847         "paddd                   %%mm3, %%mm2       \n\t"
1848
1849         "movd                    6(%0), %%mm1       \n\t"
1850         "movd                    8(%0), %%mm3       \n\t"
1851         "add                       $12, %0          \n\t"
1852         "punpcklbw               %%mm7, %%mm1       \n\t"
1853         "punpcklbw               %%mm7, %%mm3       \n\t"
1854         "movq                    %%mm1, %%mm4       \n\t"
1855         "movq                    %%mm3, %%mm5       \n\t"
1856         "pmaddwd                  (%4), %%mm1       \n\t"
1857         "pmaddwd                 8(%4), %%mm3       \n\t"
1858         "pmaddwd                16(%4), %%mm4       \n\t"
1859         "pmaddwd                 %%mm6, %%mm5       \n\t"
1860         "paddd                   %%mm3, %%mm1       \n\t"
1861         "paddd                   %%mm5, %%mm4       \n\t"
1862
1863         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1864         "paddd                   %%mm3, %%mm0       \n\t"
1865         "paddd                   %%mm3, %%mm2       \n\t"
1866         "paddd                   %%mm3, %%mm1       \n\t"
1867         "paddd                   %%mm3, %%mm4       \n\t"
1868         "psrad                     $15, %%mm0       \n\t"
1869         "psrad                     $15, %%mm2       \n\t"
1870         "psrad                     $15, %%mm1       \n\t"
1871         "psrad                     $15, %%mm4       \n\t"
1872         "packssdw                %%mm1, %%mm0       \n\t"
1873         "packssdw                %%mm4, %%mm2       \n\t"
1874         "packuswb                %%mm0, %%mm0       \n\t"
1875         "packuswb                %%mm2, %%mm2       \n\t"
1876         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1877         "movd                %%mm2, (%2, %%"REG_a") \n\t"
1878         "add                        $4, %%"REG_a"   \n\t"
1879         " js                        1b              \n\t"
1880     : "+r" (src)
1881     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1882     : "%"REG_a
1883     );
1884 }
1885
1886 static inline void RENAME(bgr24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1887 {
1888     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1889 }
1890
1891 static inline void RENAME(bgr24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1892 {
1893     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1894     assert(src1 == src2);
1895 }
1896
1897 static inline void RENAME(rgb24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1898 {
1899     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1900 }
1901
1902 static inline void RENAME(rgb24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1903 {
1904     assert(src1==src2);
1905     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1906 }
1907
1908 #if !COMPILE_TEMPLATE_MMX2
1909 // bilinear / bicubic scaling
1910 static inline void RENAME(hScale)(int16_t *dst, int dstW, const uint8_t *src, int srcW, int xInc,
1911                                   const int16_t *filter, const int16_t *filterPos, long filterSize)
1912 {
1913     assert(filterSize % 4 == 0 && filterSize>0);
1914     if (filterSize==4) { // Always true for upscaling, sometimes for down, too.
1915         x86_reg counter= -2*dstW;
1916         filter-= counter*2;
1917         filterPos-= counter/2;
1918         dst-= counter/2;
1919         __asm__ volatile(
1920 #if defined(PIC)
1921             "push            %%"REG_b"              \n\t"
1922 #endif
1923             "pxor                %%mm7, %%mm7       \n\t"
1924             "push           %%"REG_BP"              \n\t" // we use 7 regs here ...
1925             "mov             %%"REG_a", %%"REG_BP"  \n\t"
1926             ".p2align                4              \n\t"
1927             "1:                                     \n\t"
1928             "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"
1929             "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"
1930             "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"
1931             "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"
1932             "movd      (%3, %%"REG_a"), %%mm0       \n\t"
1933             "movd      (%3, %%"REG_b"), %%mm2       \n\t"
1934             "punpcklbw           %%mm7, %%mm0       \n\t"
1935             "punpcklbw           %%mm7, %%mm2       \n\t"
1936             "pmaddwd             %%mm1, %%mm0       \n\t"
1937             "pmaddwd             %%mm2, %%mm3       \n\t"
1938             "movq                %%mm0, %%mm4       \n\t"
1939             "punpckldq           %%mm3, %%mm0       \n\t"
1940             "punpckhdq           %%mm3, %%mm4       \n\t"
1941             "paddd               %%mm4, %%mm0       \n\t"
1942             "psrad                  $7, %%mm0       \n\t"
1943             "packssdw            %%mm0, %%mm0       \n\t"
1944             "movd                %%mm0, (%4, %%"REG_BP")    \n\t"
1945             "add                    $4, %%"REG_BP"  \n\t"
1946             " jnc                   1b              \n\t"
1947
1948             "pop            %%"REG_BP"              \n\t"
1949 #if defined(PIC)
1950             "pop             %%"REG_b"              \n\t"
1951 #endif
1952             : "+a" (counter)
1953             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1954 #if !defined(PIC)
1955             : "%"REG_b
1956 #endif
1957         );
1958     } else if (filterSize==8) {
1959         x86_reg counter= -2*dstW;
1960         filter-= counter*4;
1961         filterPos-= counter/2;
1962         dst-= counter/2;
1963         __asm__ volatile(
1964 #if defined(PIC)
1965             "push             %%"REG_b"             \n\t"
1966 #endif
1967             "pxor                 %%mm7, %%mm7      \n\t"
1968             "push            %%"REG_BP"             \n\t" // we use 7 regs here ...
1969             "mov              %%"REG_a", %%"REG_BP" \n\t"
1970             ".p2align                 4             \n\t"
1971             "1:                                     \n\t"
1972             "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"
1973             "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"
1974             "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"
1975             "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"
1976             "movd       (%3, %%"REG_a"), %%mm0      \n\t"
1977             "movd       (%3, %%"REG_b"), %%mm2      \n\t"
1978             "punpcklbw            %%mm7, %%mm0      \n\t"
1979             "punpcklbw            %%mm7, %%mm2      \n\t"
1980             "pmaddwd              %%mm1, %%mm0      \n\t"
1981             "pmaddwd              %%mm2, %%mm3      \n\t"
1982
1983             "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"
1984             "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"
1985             "movd      4(%3, %%"REG_a"), %%mm4      \n\t"
1986             "movd      4(%3, %%"REG_b"), %%mm2      \n\t"
1987             "punpcklbw            %%mm7, %%mm4      \n\t"
1988             "punpcklbw            %%mm7, %%mm2      \n\t"
1989             "pmaddwd              %%mm1, %%mm4      \n\t"
1990             "pmaddwd              %%mm2, %%mm5      \n\t"
1991             "paddd                %%mm4, %%mm0      \n\t"
1992             "paddd                %%mm5, %%mm3      \n\t"
1993             "movq                 %%mm0, %%mm4      \n\t"
1994             "punpckldq            %%mm3, %%mm0      \n\t"
1995             "punpckhdq            %%mm3, %%mm4      \n\t"
1996             "paddd                %%mm4, %%mm0      \n\t"
1997             "psrad                   $7, %%mm0      \n\t"
1998             "packssdw             %%mm0, %%mm0      \n\t"
1999             "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"
2000             "add                     $4, %%"REG_BP" \n\t"
2001             " jnc                    1b             \n\t"
2002
2003             "pop             %%"REG_BP"             \n\t"
2004 #if defined(PIC)
2005             "pop              %%"REG_b"             \n\t"
2006 #endif
2007             : "+a" (counter)
2008             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
2009 #if !defined(PIC)
2010             : "%"REG_b
2011 #endif
2012         );
2013     } else {
2014         const uint8_t *offset = src+filterSize;
2015         x86_reg counter= -2*dstW;
2016         //filter-= counter*filterSize/2;
2017         filterPos-= counter/2;
2018         dst-= counter/2;
2019         __asm__ volatile(
2020             "pxor                  %%mm7, %%mm7     \n\t"
2021             ".p2align                  4            \n\t"
2022             "1:                                     \n\t"
2023             "mov                      %2, %%"REG_c" \n\t"
2024             "movzwl      (%%"REG_c", %0), %%eax     \n\t"
2025             "movzwl     2(%%"REG_c", %0), %%edx     \n\t"
2026             "mov                      %5, %%"REG_c" \n\t"
2027             "pxor                  %%mm4, %%mm4     \n\t"
2028             "pxor                  %%mm5, %%mm5     \n\t"
2029             "2:                                     \n\t"
2030             "movq                   (%1), %%mm1     \n\t"
2031             "movq               (%1, %6), %%mm3     \n\t"
2032             "movd (%%"REG_c", %%"REG_a"), %%mm0     \n\t"
2033             "movd (%%"REG_c", %%"REG_d"), %%mm2     \n\t"
2034             "punpcklbw             %%mm7, %%mm0     \n\t"
2035             "punpcklbw             %%mm7, %%mm2     \n\t"
2036             "pmaddwd               %%mm1, %%mm0     \n\t"
2037             "pmaddwd               %%mm2, %%mm3     \n\t"
2038             "paddd                 %%mm3, %%mm5     \n\t"
2039             "paddd                 %%mm0, %%mm4     \n\t"
2040             "add                      $8, %1        \n\t"
2041             "add                      $4, %%"REG_c" \n\t"
2042             "cmp                      %4, %%"REG_c" \n\t"
2043             " jb                      2b            \n\t"
2044             "add                      %6, %1        \n\t"
2045             "movq                  %%mm4, %%mm0     \n\t"
2046             "punpckldq             %%mm5, %%mm4     \n\t"
2047             "punpckhdq             %%mm5, %%mm0     \n\t"
2048             "paddd                 %%mm0, %%mm4     \n\t"
2049             "psrad                    $7, %%mm4     \n\t"
2050             "packssdw              %%mm4, %%mm4     \n\t"
2051             "mov                      %3, %%"REG_a" \n\t"
2052             "movd                  %%mm4, (%%"REG_a", %0)   \n\t"
2053             "add                      $4, %0        \n\t"
2054             " jnc                     1b            \n\t"
2055
2056             : "+r" (counter), "+r" (filter)
2057             : "m" (filterPos), "m" (dst), "m"(offset),
2058             "m" (src), "r" ((x86_reg)filterSize*2)
2059             : "%"REG_a, "%"REG_c, "%"REG_d
2060         );
2061     }
2062 }
2063 #endif /* !COMPILE_TEMPLATE_MMX2 */
2064
2065 #if COMPILE_TEMPLATE_MMX2
2066 static inline void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
2067                                         long dstWidth, const uint8_t *src, int srcW,
2068                                         int xInc)
2069 {
2070     int32_t *filterPos = c->hLumFilterPos;
2071     int16_t *filter    = c->hLumFilter;
2072     int     canMMX2BeUsed  = c->canMMX2BeUsed;
2073     void    *mmx2FilterCode= c->lumMmx2FilterCode;
2074     int i;
2075 #if defined(PIC)
2076     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2077 #endif
2078
2079     __asm__ volatile(
2080 #if defined(PIC)
2081         "mov               %%"REG_b", %5        \n\t"
2082 #endif
2083         "pxor                  %%mm7, %%mm7     \n\t"
2084         "mov                      %0, %%"REG_c" \n\t"
2085         "mov                      %1, %%"REG_D" \n\t"
2086         "mov                      %2, %%"REG_d" \n\t"
2087         "mov                      %3, %%"REG_b" \n\t"
2088         "xor               %%"REG_a", %%"REG_a" \n\t" // i
2089         PREFETCH"        (%%"REG_c")            \n\t"
2090         PREFETCH"      32(%%"REG_c")            \n\t"
2091         PREFETCH"      64(%%"REG_c")            \n\t"
2092
2093 #if ARCH_X86_64
2094 #define CALL_MMX2_FILTER_CODE \
2095         "movl            (%%"REG_b"), %%esi     \n\t"\
2096         "call                    *%4            \n\t"\
2097         "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
2098         "add               %%"REG_S", %%"REG_c" \n\t"\
2099         "add               %%"REG_a", %%"REG_D" \n\t"\
2100         "xor               %%"REG_a", %%"REG_a" \n\t"\
2101
2102 #else
2103 #define CALL_MMX2_FILTER_CODE \
2104         "movl (%%"REG_b"), %%esi        \n\t"\
2105         "call         *%4                       \n\t"\
2106         "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
2107         "add               %%"REG_a", %%"REG_D" \n\t"\
2108         "xor               %%"REG_a", %%"REG_a" \n\t"\
2109
2110 #endif /* ARCH_X86_64 */
2111
2112         CALL_MMX2_FILTER_CODE
2113         CALL_MMX2_FILTER_CODE
2114         CALL_MMX2_FILTER_CODE
2115         CALL_MMX2_FILTER_CODE
2116         CALL_MMX2_FILTER_CODE
2117         CALL_MMX2_FILTER_CODE
2118         CALL_MMX2_FILTER_CODE
2119         CALL_MMX2_FILTER_CODE
2120
2121 #if defined(PIC)
2122         "mov                      %5, %%"REG_b" \n\t"
2123 #endif
2124         :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
2125            "m" (mmx2FilterCode)
2126 #if defined(PIC)
2127           ,"m" (ebxsave)
2128 #endif
2129         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2130 #if !defined(PIC)
2131          ,"%"REG_b
2132 #endif
2133     );
2134
2135     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
2136         dst[i] = src[srcW-1]*128;
2137 }
2138
2139 static inline void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2,
2140                                         long dstWidth, const uint8_t *src1,
2141                                         const uint8_t *src2, int srcW, int xInc)
2142 {
2143     int32_t *filterPos = c->hChrFilterPos;
2144     int16_t *filter    = c->hChrFilter;
2145     int     canMMX2BeUsed  = c->canMMX2BeUsed;
2146     void    *mmx2FilterCode= c->chrMmx2FilterCode;
2147     int i;
2148 #if defined(PIC)
2149     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2150 #endif
2151
2152     __asm__ volatile(
2153 #if defined(PIC)
2154         "mov          %%"REG_b", %7         \n\t"
2155 #endif
2156         "pxor             %%mm7, %%mm7      \n\t"
2157         "mov                 %0, %%"REG_c"  \n\t"
2158         "mov                 %1, %%"REG_D"  \n\t"
2159         "mov                 %2, %%"REG_d"  \n\t"
2160         "mov                 %3, %%"REG_b"  \n\t"
2161         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2162         PREFETCH"   (%%"REG_c")             \n\t"
2163         PREFETCH" 32(%%"REG_c")             \n\t"
2164         PREFETCH" 64(%%"REG_c")             \n\t"
2165
2166         CALL_MMX2_FILTER_CODE
2167         CALL_MMX2_FILTER_CODE
2168         CALL_MMX2_FILTER_CODE
2169         CALL_MMX2_FILTER_CODE
2170         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2171         "mov                 %5, %%"REG_c"  \n\t" // src
2172         "mov                 %6, %%"REG_D"  \n\t" // buf2
2173         PREFETCH"   (%%"REG_c")             \n\t"
2174         PREFETCH" 32(%%"REG_c")             \n\t"
2175         PREFETCH" 64(%%"REG_c")             \n\t"
2176
2177         CALL_MMX2_FILTER_CODE
2178         CALL_MMX2_FILTER_CODE
2179         CALL_MMX2_FILTER_CODE
2180         CALL_MMX2_FILTER_CODE
2181
2182 #if defined(PIC)
2183         "mov %7, %%"REG_b"    \n\t"
2184 #endif
2185         :: "m" (src1), "m" (dst1), "m" (filter), "m" (filterPos),
2186            "m" (mmx2FilterCode), "m" (src2), "m"(dst2)
2187 #if defined(PIC)
2188           ,"m" (ebxsave)
2189 #endif
2190         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2191 #if !defined(PIC)
2192          ,"%"REG_b
2193 #endif
2194     );
2195
2196     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
2197         dst1[i] = src1[srcW-1]*128;
2198         dst2[i] = src2[srcW-1]*128;
2199     }
2200 }
2201 #endif /* COMPILE_TEMPLATE_MMX2 */
2202
2203 #if !COMPILE_TEMPLATE_MMX2
2204 static void updateMMXDitherTables(SwsContext *c, int dstY, int lumBufIndex, int chrBufIndex,
2205                                   int lastInLumBuf, int lastInChrBuf)
2206 {
2207     const int dstH= c->dstH;
2208     const int flags= c->flags;
2209     int16_t **lumPixBuf= c->lumPixBuf;
2210     int16_t **chrUPixBuf= c->chrUPixBuf;
2211     int16_t **chrVPixBuf= c->chrVPixBuf;
2212     int16_t **alpPixBuf= c->alpPixBuf;
2213     const int vLumBufSize= c->vLumBufSize;
2214     const int vChrBufSize= c->vChrBufSize;
2215     int16_t *vLumFilterPos= c->vLumFilterPos;
2216     int16_t *vChrFilterPos= c->vChrFilterPos;
2217     int16_t *vLumFilter= c->vLumFilter;
2218     int16_t *vChrFilter= c->vChrFilter;
2219     int32_t *lumMmxFilter= c->lumMmxFilter;
2220     int32_t *chrMmxFilter= c->chrMmxFilter;
2221     int32_t av_unused *alpMmxFilter= c->alpMmxFilter;
2222     const int vLumFilterSize= c->vLumFilterSize;
2223     const int vChrFilterSize= c->vChrFilterSize;
2224     const int chrDstY= dstY>>c->chrDstVSubSample;
2225     const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input
2226     const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input
2227
2228     c->blueDither= ff_dither8[dstY&1];
2229     if (c->dstFormat == PIX_FMT_RGB555 || c->dstFormat == PIX_FMT_BGR555)
2230         c->greenDither= ff_dither8[dstY&1];
2231     else
2232         c->greenDither= ff_dither4[dstY&1];
2233     c->redDither= ff_dither8[(dstY+1)&1];
2234     if (dstY < dstH - 2) {
2235         const int16_t **lumSrcPtr= (const int16_t **) lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2236         const int16_t **chrUSrcPtr= (const int16_t **) chrUPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2237         const int16_t **chrVSrcPtr= (const int16_t **) chrVPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2238         const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;
2239         int i;
2240         if (flags & SWS_ACCURATE_RND) {
2241             int s= APCK_SIZE / 8;
2242             for (i=0; i<vLumFilterSize; i+=2) {
2243                 *(const void**)&lumMmxFilter[s*i              ]= lumSrcPtr[i  ];
2244                 *(const void**)&lumMmxFilter[s*i+APCK_PTR2/4  ]= lumSrcPtr[i+(vLumFilterSize>1)];
2245                 lumMmxFilter[s*i+APCK_COEF/4  ]=
2246                 lumMmxFilter[s*i+APCK_COEF/4+1]= vLumFilter[dstY*vLumFilterSize + i    ]
2247                            + (vLumFilterSize>1 ? vLumFilter[dstY*vLumFilterSize + i + 1]<<16 : 0);
2248                 if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2249                     *(const void**)&alpMmxFilter[s*i              ]= alpSrcPtr[i  ];
2250                     *(const void**)&alpMmxFilter[s*i+APCK_PTR2/4  ]= alpSrcPtr[i+(vLumFilterSize>1)];
2251                     alpMmxFilter[s*i+APCK_COEF/4  ]=
2252                     alpMmxFilter[s*i+APCK_COEF/4+1]= lumMmxFilter[s*i+APCK_COEF/4  ];
2253                 }
2254             }
2255             for (i=0; i<vChrFilterSize; i+=2) {
2256                 *(const void**)&chrMmxFilter[s*i              ]= chrUSrcPtr[i  ];
2257                 *(const void**)&chrMmxFilter[s*i+APCK_PTR2/4  ]= chrUSrcPtr[i+(vChrFilterSize>1)];
2258                 chrMmxFilter[s*i+APCK_COEF/4  ]=
2259                 chrMmxFilter[s*i+APCK_COEF/4+1]= vChrFilter[chrDstY*vChrFilterSize + i    ]
2260                            + (vChrFilterSize>1 ? vChrFilter[chrDstY*vChrFilterSize + i + 1]<<16 : 0);
2261             }
2262         } else {
2263             for (i=0; i<vLumFilterSize; i++) {
2264                 *(const void**)&lumMmxFilter[4*i+0]= lumSrcPtr[i];
2265                 lumMmxFilter[4*i+2]=
2266                 lumMmxFilter[4*i+3]=
2267                     ((uint16_t)vLumFilter[dstY*vLumFilterSize + i])*0x10001;
2268                 if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2269                     *(const void**)&alpMmxFilter[4*i+0]= alpSrcPtr[i];
2270                     alpMmxFilter[4*i+2]=
2271                     alpMmxFilter[4*i+3]= lumMmxFilter[4*i+2];
2272                 }
2273             }
2274             for (i=0; i<vChrFilterSize; i++) {
2275                 *(const void**)&chrMmxFilter[4*i+0]= chrUSrcPtr[i];
2276                 chrMmxFilter[4*i+2]=
2277                 chrMmxFilter[4*i+3]=
2278                     ((uint16_t)vChrFilter[chrDstY*vChrFilterSize + i])*0x10001;
2279             }
2280         }
2281     }
2282 }
2283 #endif /* !COMPILE_TEMPLATE_MMX2 */
2284
2285 static void RENAME(sws_init_swScale)(SwsContext *c)
2286 {
2287     enum PixelFormat srcFormat = c->srcFormat;
2288
2289     if (!(c->flags & SWS_BITEXACT)) {
2290         if (c->flags & SWS_ACCURATE_RND) {
2291             c->yuv2yuv1     = RENAME(yuv2yuv1_ar    );
2292             c->yuv2yuvX     = RENAME(yuv2yuvX_ar    );
2293             switch (c->dstFormat) {
2294             case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;
2295             case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;
2296             case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;
2297             case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;
2298             case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;
2299             default: break;
2300             }
2301         } else {
2302             c->yuv2yuv1     = RENAME(yuv2yuv1    );
2303             c->yuv2yuvX     = RENAME(yuv2yuvX    );
2304             switch (c->dstFormat) {
2305             case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;
2306             case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;
2307             case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;
2308             case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;
2309             case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;
2310             default: break;
2311             }
2312         }
2313         switch (c->dstFormat) {
2314         case PIX_FMT_RGB32:
2315                 c->yuv2packed1 = RENAME(yuv2rgb32_1);
2316                 c->yuv2packed2 = RENAME(yuv2rgb32_2);
2317                 break;
2318         case PIX_FMT_BGR24:
2319                 c->yuv2packed1 = RENAME(yuv2bgr24_1);
2320                 c->yuv2packed2 = RENAME(yuv2bgr24_2);
2321                 break;
2322         case PIX_FMT_RGB555:
2323                 c->yuv2packed1 = RENAME(yuv2rgb555_1);
2324                 c->yuv2packed2 = RENAME(yuv2rgb555_2);
2325                 break;
2326         case PIX_FMT_RGB565:
2327                 c->yuv2packed1 = RENAME(yuv2rgb565_1);
2328                 c->yuv2packed2 = RENAME(yuv2rgb565_2);
2329                 break;
2330         case PIX_FMT_YUYV422:
2331                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);
2332                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);
2333                 break;
2334         default:
2335                 break;
2336         }
2337     }
2338
2339 #if !COMPILE_TEMPLATE_MMX2
2340     c->hScale       = RENAME(hScale      );
2341 #endif /* !COMPILE_TEMPLATE_MMX2 */
2342
2343     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2344 #if COMPILE_TEMPLATE_MMX2
2345     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
2346     {
2347         c->hyscale_fast = RENAME(hyscale_fast);
2348         c->hcscale_fast = RENAME(hcscale_fast);
2349     } else {
2350 #endif /* COMPILE_TEMPLATE_MMX2 */
2351         c->hyscale_fast = NULL;
2352         c->hcscale_fast = NULL;
2353 #if COMPILE_TEMPLATE_MMX2
2354     }
2355 #endif /* COMPILE_TEMPLATE_MMX2 */
2356
2357 #if !COMPILE_TEMPLATE_MMX2
2358     switch(srcFormat) {
2359         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;
2360         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;
2361         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;
2362         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;
2363         case PIX_FMT_YUV420P16BE:
2364         case PIX_FMT_YUV422P16BE:
2365         case PIX_FMT_YUV444P16BE: c->chrToYV12 = RENAME(BEToUV); break;
2366         case PIX_FMT_YUV420P16LE:
2367         case PIX_FMT_YUV422P16LE:
2368         case PIX_FMT_YUV444P16LE: c->chrToYV12 = RENAME(LEToUV); break;
2369         default: break;
2370     }
2371 #endif /* !COMPILE_TEMPLATE_MMX2 */
2372     if (!c->chrSrcHSubSample) {
2373         switch(srcFormat) {
2374         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
2375         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
2376         default: break;
2377         }
2378     }
2379
2380     switch (srcFormat) {
2381 #if !COMPILE_TEMPLATE_MMX2
2382     case PIX_FMT_YUYV422  :
2383     case PIX_FMT_YUV420P16BE:
2384     case PIX_FMT_YUV422P16BE:
2385     case PIX_FMT_YUV444P16BE:
2386     case PIX_FMT_Y400A    :
2387     case PIX_FMT_GRAY16BE : c->lumToYV12 = RENAME(yuy2ToY); break;
2388     case PIX_FMT_UYVY422  :
2389     case PIX_FMT_YUV420P16LE:
2390     case PIX_FMT_YUV422P16LE:
2391     case PIX_FMT_YUV444P16LE:
2392     case PIX_FMT_GRAY16LE : c->lumToYV12 = RENAME(uyvyToY); break;
2393 #endif /* !COMPILE_TEMPLATE_MMX2 */
2394     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
2395     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
2396     default: break;
2397     }
2398 #if !COMPILE_TEMPLATE_MMX2
2399     if (c->alpPixBuf) {
2400         switch (srcFormat) {
2401         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;
2402         default: break;
2403         }
2404     }
2405 #endif /* !COMPILE_TEMPLATE_MMX2 */
2406 }