swscale: fix types of assembly arguments.
[ffmpeg.git] / libswscale / x86 / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of Libav.
5  *
6  * Libav is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * Libav is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with Libav; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #undef REAL_MOVNTQ
22 #undef MOVNTQ
23 #undef PREFETCH
24
25 #if COMPILE_TEMPLATE_MMX2
26 #define PREFETCH "prefetchnta"
27 #else
28 #define PREFETCH  " # nop"
29 #endif
30
31 #if COMPILE_TEMPLATE_MMX2
32 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
33 #else
34 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
35 #endif
36 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
37
38 #define YSCALEYUV2YV12X(offset, dest, end, pos) \
39     __asm__ volatile(\
40         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
41         "movq                             %%mm3, %%mm4      \n\t"\
42         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
43         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
44         ".p2align                             4             \n\t" /* FIXME Unroll? */\
45         "1:                                                 \n\t"\
46         "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\
47         "movq                (%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\
48         "movq               8(%%"REG_S", %3, 2), %%mm5      \n\t" /* srcData */\
49         "add                                $16, %%"REG_d"  \n\t"\
50         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
51         "test                         %%"REG_S", %%"REG_S"  \n\t"\
52         "pmulhw                           %%mm0, %%mm2      \n\t"\
53         "pmulhw                           %%mm0, %%mm5      \n\t"\
54         "paddw                            %%mm2, %%mm3      \n\t"\
55         "paddw                            %%mm5, %%mm4      \n\t"\
56         " jnz                                1b             \n\t"\
57         "psraw                               $3, %%mm3      \n\t"\
58         "psraw                               $3, %%mm4      \n\t"\
59         "packuswb                         %%mm4, %%mm3      \n\t"\
60         MOVNTQ(%%mm3, (%1, %3))\
61         "add                                 $8, %3         \n\t"\
62         "cmp                                 %2, %3         \n\t"\
63         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
64         "movq                             %%mm3, %%mm4      \n\t"\
65         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
66         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
67         "jb                                  1b             \n\t"\
68         :: "r" (&c->redDither),\
69            "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\
70         : "%"REG_d, "%"REG_S\
71     );
72
73 static inline void RENAME(yuv2yuvX)(SwsContext *c, const int16_t *lumFilter,
74                                     const int16_t **lumSrc, int lumFilterSize,
75                                     const int16_t *chrFilter, const int16_t **chrUSrc,
76                                     const int16_t **chrVSrc,
77                                     int chrFilterSize, const int16_t **alpSrc,
78                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
79                                     uint8_t *aDest, int dstW, int chrDstW)
80 {
81     if (uDest) {
82         x86_reg uv_off = c->uv_off;
83         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)
84         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, vDest - uv_off, chrDstW + uv_off, uv_off)
85     }
86     if (CONFIG_SWSCALE_ALPHA && aDest) {
87         YSCALEYUV2YV12X(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)
88     }
89
90     YSCALEYUV2YV12X(LUM_MMX_FILTER_OFFSET, dest, dstW, 0)
91 }
92
93 #define YSCALEYUV2YV12X_ACCURATE(offset, dest, end, pos) \
94     __asm__ volatile(\
95         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
96         "pxor                             %%mm4, %%mm4      \n\t"\
97         "pxor                             %%mm5, %%mm5      \n\t"\
98         "pxor                             %%mm6, %%mm6      \n\t"\
99         "pxor                             %%mm7, %%mm7      \n\t"\
100         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
101         ".p2align                             4             \n\t"\
102         "1:                                                 \n\t"\
103         "movq                (%%"REG_S", %3, 2), %%mm0      \n\t" /* srcData */\
104         "movq               8(%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\
105         "mov        "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"  \n\t"\
106         "movq                (%%"REG_S", %3, 2), %%mm1      \n\t" /* srcData */\
107         "movq                             %%mm0, %%mm3      \n\t"\
108         "punpcklwd                        %%mm1, %%mm0      \n\t"\
109         "punpckhwd                        %%mm1, %%mm3      \n\t"\
110         "movq       "STR(APCK_COEF)"(%%"REG_d"), %%mm1      \n\t" /* filterCoeff */\
111         "pmaddwd                          %%mm1, %%mm0      \n\t"\
112         "pmaddwd                          %%mm1, %%mm3      \n\t"\
113         "paddd                            %%mm0, %%mm4      \n\t"\
114         "paddd                            %%mm3, %%mm5      \n\t"\
115         "movq               8(%%"REG_S", %3, 2), %%mm3      \n\t" /* srcData */\
116         "mov        "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"  \n\t"\
117         "add                  $"STR(APCK_SIZE)", %%"REG_d"  \n\t"\
118         "test                         %%"REG_S", %%"REG_S"  \n\t"\
119         "movq                             %%mm2, %%mm0      \n\t"\
120         "punpcklwd                        %%mm3, %%mm2      \n\t"\
121         "punpckhwd                        %%mm3, %%mm0      \n\t"\
122         "pmaddwd                          %%mm1, %%mm2      \n\t"\
123         "pmaddwd                          %%mm1, %%mm0      \n\t"\
124         "paddd                            %%mm2, %%mm6      \n\t"\
125         "paddd                            %%mm0, %%mm7      \n\t"\
126         " jnz                                1b             \n\t"\
127         "psrad                              $16, %%mm4      \n\t"\
128         "psrad                              $16, %%mm5      \n\t"\
129         "psrad                              $16, %%mm6      \n\t"\
130         "psrad                              $16, %%mm7      \n\t"\
131         "movq             "VROUNDER_OFFSET"(%0), %%mm0      \n\t"\
132         "packssdw                         %%mm5, %%mm4      \n\t"\
133         "packssdw                         %%mm7, %%mm6      \n\t"\
134         "paddw                            %%mm0, %%mm4      \n\t"\
135         "paddw                            %%mm0, %%mm6      \n\t"\
136         "psraw                               $3, %%mm4      \n\t"\
137         "psraw                               $3, %%mm6      \n\t"\
138         "packuswb                         %%mm6, %%mm4      \n\t"\
139         MOVNTQ(%%mm4, (%1, %3))\
140         "add                                 $8, %3         \n\t"\
141         "cmp                                 %2, %3         \n\t"\
142         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
143         "pxor                             %%mm4, %%mm4      \n\t"\
144         "pxor                             %%mm5, %%mm5      \n\t"\
145         "pxor                             %%mm6, %%mm6      \n\t"\
146         "pxor                             %%mm7, %%mm7      \n\t"\
147         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
148         "jb                                  1b             \n\t"\
149         :: "r" (&c->redDither),\
150         "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\
151         : "%"REG_a, "%"REG_d, "%"REG_S\
152     );
153
154 static inline void RENAME(yuv2yuvX_ar)(SwsContext *c, const int16_t *lumFilter,
155                                        const int16_t **lumSrc, int lumFilterSize,
156                                        const int16_t *chrFilter, const int16_t **chrUSrc,
157                                        const int16_t **chrVSrc,
158                                        int chrFilterSize, const int16_t **alpSrc,
159                                        uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
160                                        uint8_t *aDest, int dstW, int chrDstW)
161 {
162     if (uDest) {
163         x86_reg uv_off = c->uv_off;
164         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)
165         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, vDest - uv_off, chrDstW + uv_off, uv_off)
166     }
167     if (CONFIG_SWSCALE_ALPHA && aDest) {
168         YSCALEYUV2YV12X_ACCURATE(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)
169     }
170
171     YSCALEYUV2YV12X_ACCURATE(LUM_MMX_FILTER_OFFSET, dest, dstW, 0)
172 }
173
174 static inline void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc,
175                                     const int16_t *chrUSrc, const int16_t *chrVSrc,
176                                     const int16_t *alpSrc,
177                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
178                                     uint8_t *aDest, int dstW, int chrDstW)
179 {
180     int p= 4;
181     const int16_t *src[4]= { alpSrc + dstW, lumSrc + dstW, chrUSrc + chrDstW, chrVSrc + chrDstW };
182     uint8_t *dst[4]= { aDest, dest, uDest, vDest };
183     x86_reg counter[4]= { dstW, dstW, chrDstW, chrDstW };
184
185     while (p--) {
186         if (dst[p]) {
187             __asm__ volatile(
188                 "mov %2, %%"REG_a"                    \n\t"
189                 ".p2align               4             \n\t" /* FIXME Unroll? */
190                 "1:                                   \n\t"
191                 "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"
192                 "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"
193                 "psraw                 $7, %%mm0      \n\t"
194                 "psraw                 $7, %%mm1      \n\t"
195                 "packuswb           %%mm1, %%mm0      \n\t"
196                 MOVNTQ(%%mm0, (%1, %%REGa))
197                 "add                   $8, %%"REG_a"  \n\t"
198                 "jnc                   1b             \n\t"
199                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
200                    "g" (-counter[p])
201                 : "%"REG_a
202             );
203         }
204     }
205 }
206
207 static inline void RENAME(yuv2yuv1_ar)(SwsContext *c, const int16_t *lumSrc,
208                                        const int16_t *chrUSrc, const int16_t *chrVSrc,
209                                        const int16_t *alpSrc,
210                                        uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
211                                        uint8_t *aDest, int dstW, int chrDstW)
212 {
213     int p= 4;
214     const int16_t *src[4]= { alpSrc + dstW, lumSrc + dstW, chrUSrc + chrDstW, chrVSrc + chrDstW };
215     uint8_t *dst[4]= { aDest, dest, uDest, vDest };
216     x86_reg counter[4]= { dstW, dstW, chrDstW, chrDstW };
217
218     while (p--) {
219         if (dst[p]) {
220             __asm__ volatile(
221                 "mov %2, %%"REG_a"                    \n\t"
222                 "pcmpeqw %%mm7, %%mm7                 \n\t"
223                 "psrlw                 $15, %%mm7     \n\t"
224                 "psllw                  $6, %%mm7     \n\t"
225                 ".p2align                4            \n\t" /* FIXME Unroll? */
226                 "1:                                   \n\t"
227                 "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"
228                 "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"
229                 "paddsw             %%mm7, %%mm0      \n\t"
230                 "paddsw             %%mm7, %%mm1      \n\t"
231                 "psraw                 $7, %%mm0      \n\t"
232                 "psraw                 $7, %%mm1      \n\t"
233                 "packuswb           %%mm1, %%mm0      \n\t"
234                 MOVNTQ(%%mm0, (%1, %%REGa))
235                 "add                   $8, %%"REG_a"  \n\t"
236                 "jnc                   1b             \n\t"
237                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
238                    "g" (-counter[p])
239                 : "%"REG_a
240             );
241         }
242     }
243 }
244
245 #define YSCALEYUV2PACKEDX_UV \
246     __asm__ volatile(\
247         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
248         ".p2align                      4                \n\t"\
249         "nop                                            \n\t"\
250         "1:                                             \n\t"\
251         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
252         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
253         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
254         "movq                      %%mm3, %%mm4         \n\t"\
255         ".p2align                      4                \n\t"\
256         "2:                                             \n\t"\
257         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
258         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
259         "add                          %6, %%"REG_S"     \n\t" \
260         "movq     (%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
261         "add                         $16, %%"REG_d"     \n\t"\
262         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
263         "pmulhw                    %%mm0, %%mm2         \n\t"\
264         "pmulhw                    %%mm0, %%mm5         \n\t"\
265         "paddw                     %%mm2, %%mm3         \n\t"\
266         "paddw                     %%mm5, %%mm4         \n\t"\
267         "test                  %%"REG_S", %%"REG_S"     \n\t"\
268         " jnz                         2b                \n\t"\
269
270 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
271     "lea                "offset"(%0), %%"REG_d"     \n\t"\
272     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
273     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
274     "movq                    "#dst1", "#dst2"       \n\t"\
275     ".p2align                      4                \n\t"\
276     "2:                                             \n\t"\
277     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
278     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
279     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
280     "add                         $16, %%"REG_d"            \n\t"\
281     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
282     "pmulhw                 "#coeff", "#src1"       \n\t"\
283     "pmulhw                 "#coeff", "#src2"       \n\t"\
284     "paddw                   "#src1", "#dst1"       \n\t"\
285     "paddw                   "#src2", "#dst2"       \n\t"\
286     "test                  %%"REG_S", %%"REG_S"     \n\t"\
287     " jnz                         2b                \n\t"\
288
289 #define YSCALEYUV2PACKEDX \
290     YSCALEYUV2PACKEDX_UV \
291     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
292
293 #define YSCALEYUV2PACKEDX_END                     \
294         :: "r" (&c->redDither),                   \
295             "m" (dummy), "m" (dummy), "m" (dummy),\
296             "r" (dest), "m" (dstW_reg), "m"(uv_off) \
297         : "%"REG_a, "%"REG_d, "%"REG_S            \
298     );
299
300 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
301     __asm__ volatile(\
302         "xor %%"REG_a", %%"REG_a"                       \n\t"\
303         ".p2align                      4                \n\t"\
304         "nop                                            \n\t"\
305         "1:                                             \n\t"\
306         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
307         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
308         "pxor                      %%mm4, %%mm4         \n\t"\
309         "pxor                      %%mm5, %%mm5         \n\t"\
310         "pxor                      %%mm6, %%mm6         \n\t"\
311         "pxor                      %%mm7, %%mm7         \n\t"\
312         ".p2align                      4                \n\t"\
313         "2:                                             \n\t"\
314         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
315         "add                          %6, %%"REG_S"      \n\t" \
316         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
317         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
318         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
319         "movq                      %%mm0, %%mm3         \n\t"\
320         "punpcklwd                 %%mm1, %%mm0         \n\t"\
321         "punpckhwd                 %%mm1, %%mm3         \n\t"\
322         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
323         "pmaddwd                   %%mm1, %%mm0         \n\t"\
324         "pmaddwd                   %%mm1, %%mm3         \n\t"\
325         "paddd                     %%mm0, %%mm4         \n\t"\
326         "paddd                     %%mm3, %%mm5         \n\t"\
327         "add                          %6, %%"REG_S"      \n\t" \
328         "movq     (%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
329         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
330         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
331         "test                  %%"REG_S", %%"REG_S"     \n\t"\
332         "movq                      %%mm2, %%mm0         \n\t"\
333         "punpcklwd                 %%mm3, %%mm2         \n\t"\
334         "punpckhwd                 %%mm3, %%mm0         \n\t"\
335         "pmaddwd                   %%mm1, %%mm2         \n\t"\
336         "pmaddwd                   %%mm1, %%mm0         \n\t"\
337         "paddd                     %%mm2, %%mm6         \n\t"\
338         "paddd                     %%mm0, %%mm7         \n\t"\
339         " jnz                         2b                \n\t"\
340         "psrad                       $16, %%mm4         \n\t"\
341         "psrad                       $16, %%mm5         \n\t"\
342         "psrad                       $16, %%mm6         \n\t"\
343         "psrad                       $16, %%mm7         \n\t"\
344         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
345         "packssdw                  %%mm5, %%mm4         \n\t"\
346         "packssdw                  %%mm7, %%mm6         \n\t"\
347         "paddw                     %%mm0, %%mm4         \n\t"\
348         "paddw                     %%mm0, %%mm6         \n\t"\
349         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
350         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
351
352 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
353     "lea                "offset"(%0), %%"REG_d"     \n\t"\
354     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
355     "pxor                      %%mm1, %%mm1         \n\t"\
356     "pxor                      %%mm5, %%mm5         \n\t"\
357     "pxor                      %%mm7, %%mm7         \n\t"\
358     "pxor                      %%mm6, %%mm6         \n\t"\
359     ".p2align                      4                \n\t"\
360     "2:                                             \n\t"\
361     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
362     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
363     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
364     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
365     "movq                      %%mm0, %%mm3         \n\t"\
366     "punpcklwd                 %%mm4, %%mm0         \n\t"\
367     "punpckhwd                 %%mm4, %%mm3         \n\t"\
368     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
369     "pmaddwd                   %%mm4, %%mm0         \n\t"\
370     "pmaddwd                   %%mm4, %%mm3         \n\t"\
371     "paddd                     %%mm0, %%mm1         \n\t"\
372     "paddd                     %%mm3, %%mm5         \n\t"\
373     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
374     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
375     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
376     "test                  %%"REG_S", %%"REG_S"     \n\t"\
377     "movq                      %%mm2, %%mm0         \n\t"\
378     "punpcklwd                 %%mm3, %%mm2         \n\t"\
379     "punpckhwd                 %%mm3, %%mm0         \n\t"\
380     "pmaddwd                   %%mm4, %%mm2         \n\t"\
381     "pmaddwd                   %%mm4, %%mm0         \n\t"\
382     "paddd                     %%mm2, %%mm7         \n\t"\
383     "paddd                     %%mm0, %%mm6         \n\t"\
384     " jnz                         2b                \n\t"\
385     "psrad                       $16, %%mm1         \n\t"\
386     "psrad                       $16, %%mm5         \n\t"\
387     "psrad                       $16, %%mm7         \n\t"\
388     "psrad                       $16, %%mm6         \n\t"\
389     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
390     "packssdw                  %%mm5, %%mm1         \n\t"\
391     "packssdw                  %%mm6, %%mm7         \n\t"\
392     "paddw                     %%mm0, %%mm1         \n\t"\
393     "paddw                     %%mm0, %%mm7         \n\t"\
394     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
395     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
396
397 #define YSCALEYUV2PACKEDX_ACCURATE \
398     YSCALEYUV2PACKEDX_ACCURATE_UV \
399     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
400
401 #define YSCALEYUV2RGBX \
402     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
403     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
404     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
405     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
406     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
407     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
408     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
409     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
410     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
411     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
412     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
413     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
414     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
415     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
416     "paddw           %%mm3, %%mm4       \n\t"\
417     "movq            %%mm2, %%mm0       \n\t"\
418     "movq            %%mm5, %%mm6       \n\t"\
419     "movq            %%mm4, %%mm3       \n\t"\
420     "punpcklwd       %%mm2, %%mm2       \n\t"\
421     "punpcklwd       %%mm5, %%mm5       \n\t"\
422     "punpcklwd       %%mm4, %%mm4       \n\t"\
423     "paddw           %%mm1, %%mm2       \n\t"\
424     "paddw           %%mm1, %%mm5       \n\t"\
425     "paddw           %%mm1, %%mm4       \n\t"\
426     "punpckhwd       %%mm0, %%mm0       \n\t"\
427     "punpckhwd       %%mm6, %%mm6       \n\t"\
428     "punpckhwd       %%mm3, %%mm3       \n\t"\
429     "paddw           %%mm7, %%mm0       \n\t"\
430     "paddw           %%mm7, %%mm6       \n\t"\
431     "paddw           %%mm7, %%mm3       \n\t"\
432     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
433     "packuswb        %%mm0, %%mm2       \n\t"\
434     "packuswb        %%mm6, %%mm5       \n\t"\
435     "packuswb        %%mm3, %%mm4       \n\t"\
436
437 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
438     "movq       "#b", "#q2"     \n\t" /* B */\
439     "movq       "#r", "#t"      \n\t" /* R */\
440     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
441     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
442     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
443     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
444     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
445     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
446     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
447     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
448     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
449     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
450 \
451     MOVNTQ(   q0,   (dst, index, 4))\
452     MOVNTQ(    b,  8(dst, index, 4))\
453     MOVNTQ(   q2, 16(dst, index, 4))\
454     MOVNTQ(   q3, 24(dst, index, 4))\
455 \
456     "add      $8, "#index"      \n\t"\
457     "cmp "#dstw", "#index"      \n\t"\
458     " jb      1b                \n\t"
459 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
460
461 static inline void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,
462                                           const int16_t **lumSrc, int lumFilterSize,
463                                           const int16_t *chrFilter, const int16_t **chrUSrc,
464                                           const int16_t **chrVSrc,
465                                           int chrFilterSize, const int16_t **alpSrc,
466                                           uint8_t *dest, int dstW, int dstY)
467 {
468     x86_reg dummy=0;
469     x86_reg dstW_reg = dstW;
470     x86_reg uv_off = c->uv_off << 1;
471
472     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
473         YSCALEYUV2PACKEDX_ACCURATE
474         YSCALEYUV2RGBX
475         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
476         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
477         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
478         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
479         "movq               "Y_TEMP"(%0), %%mm5         \n\t"
480         "psraw                        $3, %%mm1         \n\t"
481         "psraw                        $3, %%mm7         \n\t"
482         "packuswb                  %%mm7, %%mm1         \n\t"
483         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
484         YSCALEYUV2PACKEDX_END
485     } else {
486         YSCALEYUV2PACKEDX_ACCURATE
487         YSCALEYUV2RGBX
488         "pcmpeqd %%mm7, %%mm7 \n\t"
489         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
490         YSCALEYUV2PACKEDX_END
491     }
492 }
493
494 static inline void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,
495                                        const int16_t **lumSrc, int lumFilterSize,
496                                        const int16_t *chrFilter, const int16_t **chrUSrc,
497                                        const int16_t **chrVSrc,
498                                        int chrFilterSize, const int16_t **alpSrc,
499                                        uint8_t *dest, int dstW, int dstY)
500 {
501     x86_reg dummy=0;
502     x86_reg dstW_reg = dstW;
503     x86_reg uv_off = c->uv_off << 1;
504
505     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
506         YSCALEYUV2PACKEDX
507         YSCALEYUV2RGBX
508         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
509         "psraw                        $3, %%mm1         \n\t"
510         "psraw                        $3, %%mm7         \n\t"
511         "packuswb                  %%mm7, %%mm1         \n\t"
512         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
513         YSCALEYUV2PACKEDX_END
514     } else {
515         YSCALEYUV2PACKEDX
516         YSCALEYUV2RGBX
517         "pcmpeqd %%mm7, %%mm7 \n\t"
518         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
519         YSCALEYUV2PACKEDX_END
520     }
521 }
522
523 #define REAL_WRITERGB16(dst, dstw, index) \
524     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
525     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
526     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
527     "psrlq           $3, %%mm2  \n\t"\
528 \
529     "movq         %%mm2, %%mm1  \n\t"\
530     "movq         %%mm4, %%mm3  \n\t"\
531 \
532     "punpcklbw    %%mm7, %%mm3  \n\t"\
533     "punpcklbw    %%mm5, %%mm2  \n\t"\
534     "punpckhbw    %%mm7, %%mm4  \n\t"\
535     "punpckhbw    %%mm5, %%mm1  \n\t"\
536 \
537     "psllq           $3, %%mm3  \n\t"\
538     "psllq           $3, %%mm4  \n\t"\
539 \
540     "por          %%mm3, %%mm2  \n\t"\
541     "por          %%mm4, %%mm1  \n\t"\
542 \
543     MOVNTQ(%%mm2,  (dst, index, 2))\
544     MOVNTQ(%%mm1, 8(dst, index, 2))\
545 \
546     "add             $8, "#index"   \n\t"\
547     "cmp        "#dstw", "#index"   \n\t"\
548     " jb             1b             \n\t"
549 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
550
551 static inline void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,
552                                            const int16_t **lumSrc, int lumFilterSize,
553                                            const int16_t *chrFilter, const int16_t **chrUSrc,
554                                            const int16_t **chrVSrc,
555                                            int chrFilterSize, const int16_t **alpSrc,
556                                            uint8_t *dest, int dstW, int dstY)
557 {
558     x86_reg dummy=0;
559     x86_reg dstW_reg = dstW;
560     x86_reg uv_off = c->uv_off << 1;
561
562     YSCALEYUV2PACKEDX_ACCURATE
563     YSCALEYUV2RGBX
564     "pxor %%mm7, %%mm7 \n\t"
565     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
566 #ifdef DITHER1XBPP
567     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
568     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
569     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
570 #endif
571     WRITERGB16(%4, %5, %%REGa)
572     YSCALEYUV2PACKEDX_END
573 }
574
575 static inline void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,
576                                         const int16_t **lumSrc, int lumFilterSize,
577                                         const int16_t *chrFilter, const int16_t **chrUSrc,
578                                         const int16_t **chrVSrc,
579                                         int chrFilterSize, const int16_t **alpSrc,
580                                         uint8_t *dest, int dstW, int dstY)
581 {
582     x86_reg dummy=0;
583     x86_reg dstW_reg = dstW;
584     x86_reg uv_off = c->uv_off << 1;
585
586     YSCALEYUV2PACKEDX
587     YSCALEYUV2RGBX
588     "pxor %%mm7, %%mm7 \n\t"
589     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
590 #ifdef DITHER1XBPP
591     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
592     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
593     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
594 #endif
595     WRITERGB16(%4, %5, %%REGa)
596     YSCALEYUV2PACKEDX_END
597 }
598
599 #define REAL_WRITERGB15(dst, dstw, index) \
600     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
601     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
602     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
603     "psrlq           $3, %%mm2  \n\t"\
604     "psrlq           $1, %%mm5  \n\t"\
605 \
606     "movq         %%mm2, %%mm1  \n\t"\
607     "movq         %%mm4, %%mm3  \n\t"\
608 \
609     "punpcklbw    %%mm7, %%mm3  \n\t"\
610     "punpcklbw    %%mm5, %%mm2  \n\t"\
611     "punpckhbw    %%mm7, %%mm4  \n\t"\
612     "punpckhbw    %%mm5, %%mm1  \n\t"\
613 \
614     "psllq           $2, %%mm3  \n\t"\
615     "psllq           $2, %%mm4  \n\t"\
616 \
617     "por          %%mm3, %%mm2  \n\t"\
618     "por          %%mm4, %%mm1  \n\t"\
619 \
620     MOVNTQ(%%mm2,  (dst, index, 2))\
621     MOVNTQ(%%mm1, 8(dst, index, 2))\
622 \
623     "add             $8, "#index"   \n\t"\
624     "cmp        "#dstw", "#index"   \n\t"\
625     " jb             1b             \n\t"
626 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
627
628 static inline void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,
629                                            const int16_t **lumSrc, int lumFilterSize,
630                                            const int16_t *chrFilter, const int16_t **chrUSrc,
631                                            const int16_t **chrVSrc,
632                                            int chrFilterSize, const int16_t **alpSrc,
633                                            uint8_t *dest, int dstW, int dstY)
634 {
635     x86_reg dummy=0;
636     x86_reg dstW_reg = dstW;
637     x86_reg uv_off = c->uv_off << 1;
638
639     YSCALEYUV2PACKEDX_ACCURATE
640     YSCALEYUV2RGBX
641     "pxor %%mm7, %%mm7 \n\t"
642     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
643 #ifdef DITHER1XBPP
644     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
645     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
646     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
647 #endif
648     WRITERGB15(%4, %5, %%REGa)
649     YSCALEYUV2PACKEDX_END
650 }
651
652 static inline void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,
653                                         const int16_t **lumSrc, int lumFilterSize,
654                                         const int16_t *chrFilter, const int16_t **chrUSrc,
655                                         const int16_t **chrVSrc,
656                                         int chrFilterSize, const int16_t **alpSrc,
657                                         uint8_t *dest, int dstW, int dstY)
658 {
659     x86_reg dummy=0;
660     x86_reg dstW_reg = dstW;
661     x86_reg uv_off = c->uv_off << 1;
662
663     YSCALEYUV2PACKEDX
664     YSCALEYUV2RGBX
665     "pxor %%mm7, %%mm7 \n\t"
666     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
667 #ifdef DITHER1XBPP
668     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
669     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
670     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
671 #endif
672     WRITERGB15(%4, %5, %%REGa)
673     YSCALEYUV2PACKEDX_END
674 }
675
676 #define WRITEBGR24MMX(dst, dstw, index) \
677     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
678     "movq      %%mm2, %%mm1     \n\t" /* B */\
679     "movq      %%mm5, %%mm6     \n\t" /* R */\
680     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
681     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
682     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
683     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
684     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
685     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
686     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
687     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
688     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
689     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
690 \
691     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
692     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
693     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
694     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
695 \
696     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
697     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
698     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
699     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
700 \
701     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
702     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
703     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
704     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
705 \
706     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
707     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
708     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
709     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
710     MOVNTQ(%%mm0, (dst))\
711 \
712     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
713     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
714     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
715     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
716     MOVNTQ(%%mm6, 8(dst))\
717 \
718     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
719     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
720     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
721     MOVNTQ(%%mm5, 16(dst))\
722 \
723     "add         $24, "#dst"    \n\t"\
724 \
725     "add          $8, "#index"  \n\t"\
726     "cmp     "#dstw", "#index"  \n\t"\
727     " jb          1b            \n\t"
728
729 #define WRITEBGR24MMX2(dst, dstw, index) \
730     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
731     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
732     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
733     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
734     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
735     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
736 \
737     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
738     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
739     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
740 \
741     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
742     "por    %%mm1, %%mm6        \n\t"\
743     "por    %%mm3, %%mm6        \n\t"\
744     MOVNTQ(%%mm6, (dst))\
745 \
746     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
747     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
748     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
749     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
750 \
751     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
752     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
753     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
754 \
755     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
756     "por    %%mm3, %%mm6        \n\t"\
757     MOVNTQ(%%mm6, 8(dst))\
758 \
759     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
760     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
761     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
762 \
763     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
764     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
765     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
766 \
767     "por    %%mm1, %%mm3        \n\t"\
768     "por    %%mm3, %%mm6        \n\t"\
769     MOVNTQ(%%mm6, 16(dst))\
770 \
771     "add      $24, "#dst"       \n\t"\
772 \
773     "add       $8, "#index"     \n\t"\
774     "cmp  "#dstw", "#index"     \n\t"\
775     " jb       1b               \n\t"
776
777 #if COMPILE_TEMPLATE_MMX2
778 #undef WRITEBGR24
779 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
780 #else
781 #undef WRITEBGR24
782 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
783 #endif
784
785 static inline void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,
786                                           const int16_t **lumSrc, int lumFilterSize,
787                                           const int16_t *chrFilter, const int16_t **chrUSrc,
788                                           const int16_t **chrVSrc,
789                                           int chrFilterSize, const int16_t **alpSrc,
790                                           uint8_t *dest, int dstW, int dstY)
791 {
792     x86_reg dummy=0;
793     x86_reg dstW_reg = dstW;
794     x86_reg uv_off = c->uv_off << 1;
795
796     YSCALEYUV2PACKEDX_ACCURATE
797     YSCALEYUV2RGBX
798     "pxor %%mm7, %%mm7 \n\t"
799     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
800     "add %4, %%"REG_c"                        \n\t"
801     WRITEBGR24(%%REGc, %5, %%REGa)
802     :: "r" (&c->redDither),
803        "m" (dummy), "m" (dummy), "m" (dummy),
804        "r" (dest), "m" (dstW_reg), "m"(uv_off)
805     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
806     );
807 }
808
809 static inline void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,
810                                        const int16_t **lumSrc, int lumFilterSize,
811                                        const int16_t *chrFilter, const int16_t **chrUSrc,
812                                        const int16_t **chrVSrc,
813                                        int chrFilterSize, const int16_t **alpSrc,
814                                        uint8_t *dest, int dstW, int dstY)
815 {
816     x86_reg dummy=0;
817     x86_reg dstW_reg = dstW;
818     x86_reg uv_off = c->uv_off << 1;
819
820     YSCALEYUV2PACKEDX
821     YSCALEYUV2RGBX
822     "pxor                    %%mm7, %%mm7       \n\t"
823     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
824     "add                        %4, %%"REG_c"   \n\t"
825     WRITEBGR24(%%REGc, %5, %%REGa)
826     :: "r" (&c->redDither),
827        "m" (dummy), "m" (dummy), "m" (dummy),
828        "r" (dest),  "m" (dstW_reg), "m"(uv_off)
829     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
830     );
831 }
832
833 #define REAL_WRITEYUY2(dst, dstw, index) \
834     "packuswb  %%mm3, %%mm3     \n\t"\
835     "packuswb  %%mm4, %%mm4     \n\t"\
836     "packuswb  %%mm7, %%mm1     \n\t"\
837     "punpcklbw %%mm4, %%mm3     \n\t"\
838     "movq      %%mm1, %%mm7     \n\t"\
839     "punpcklbw %%mm3, %%mm1     \n\t"\
840     "punpckhbw %%mm3, %%mm7     \n\t"\
841 \
842     MOVNTQ(%%mm1, (dst, index, 2))\
843     MOVNTQ(%%mm7, 8(dst, index, 2))\
844 \
845     "add          $8, "#index"  \n\t"\
846     "cmp     "#dstw", "#index"  \n\t"\
847     " jb          1b            \n\t"
848 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
849
850 static inline void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,
851                                             const int16_t **lumSrc, int lumFilterSize,
852                                             const int16_t *chrFilter, const int16_t **chrUSrc,
853                                             const int16_t **chrVSrc,
854                                             int chrFilterSize, const int16_t **alpSrc,
855                                             uint8_t *dest, int dstW, int dstY)
856 {
857     x86_reg dummy=0;
858     x86_reg dstW_reg = dstW;
859     x86_reg uv_off = c->uv_off << 1;
860
861     YSCALEYUV2PACKEDX_ACCURATE
862     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
863     "psraw $3, %%mm3    \n\t"
864     "psraw $3, %%mm4    \n\t"
865     "psraw $3, %%mm1    \n\t"
866     "psraw $3, %%mm7    \n\t"
867     WRITEYUY2(%4, %5, %%REGa)
868     YSCALEYUV2PACKEDX_END
869 }
870
871 static inline void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,
872                                          const int16_t **lumSrc, int lumFilterSize,
873                                          const int16_t *chrFilter, const int16_t **chrUSrc,
874                                          const int16_t **chrVSrc,
875                                          int chrFilterSize, const int16_t **alpSrc,
876                                          uint8_t *dest, int dstW, int dstY)
877 {
878     x86_reg dummy=0;
879     x86_reg dstW_reg = dstW;
880     x86_reg uv_off = c->uv_off << 1;
881
882     YSCALEYUV2PACKEDX
883     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
884     "psraw $3, %%mm3    \n\t"
885     "psraw $3, %%mm4    \n\t"
886     "psraw $3, %%mm1    \n\t"
887     "psraw $3, %%mm7    \n\t"
888     WRITEYUY2(%4, %5, %%REGa)
889     YSCALEYUV2PACKEDX_END
890 }
891
892 #define REAL_YSCALEYUV2RGB_UV(index, c) \
893     "xor            "#index", "#index"  \n\t"\
894     ".p2align              4            \n\t"\
895     "1:                                 \n\t"\
896     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
897     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
898     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
899     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
900     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
901     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
902     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
903     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
904     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
905     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
906     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
907     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
908     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
909     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
910     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
911     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
912     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
913     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
914     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
915     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
916     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
917     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
918
919 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
920     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
921     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
922     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
923     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
924     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
925     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
926     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
927     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
928     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
929     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
930     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
931     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
932
933 #define REAL_YSCALEYUV2RGB_COEFF(c) \
934     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
935     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
936     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
937     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
938     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
939     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
940     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
941     "paddw             %%mm3, %%mm4     \n\t"\
942     "movq              %%mm2, %%mm0     \n\t"\
943     "movq              %%mm5, %%mm6     \n\t"\
944     "movq              %%mm4, %%mm3     \n\t"\
945     "punpcklwd         %%mm2, %%mm2     \n\t"\
946     "punpcklwd         %%mm5, %%mm5     \n\t"\
947     "punpcklwd         %%mm4, %%mm4     \n\t"\
948     "paddw             %%mm1, %%mm2     \n\t"\
949     "paddw             %%mm1, %%mm5     \n\t"\
950     "paddw             %%mm1, %%mm4     \n\t"\
951     "punpckhwd         %%mm0, %%mm0     \n\t"\
952     "punpckhwd         %%mm6, %%mm6     \n\t"\
953     "punpckhwd         %%mm3, %%mm3     \n\t"\
954     "paddw             %%mm7, %%mm0     \n\t"\
955     "paddw             %%mm7, %%mm6     \n\t"\
956     "paddw             %%mm7, %%mm3     \n\t"\
957     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
958     "packuswb          %%mm0, %%mm2     \n\t"\
959     "packuswb          %%mm6, %%mm5     \n\t"\
960     "packuswb          %%mm3, %%mm4     \n\t"\
961
962 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
963
964 #define YSCALEYUV2RGB(index, c) \
965     REAL_YSCALEYUV2RGB_UV(index, c) \
966     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
967     REAL_YSCALEYUV2RGB_COEFF(c)
968
969 /**
970  * vertical bilinear scale YV12 to RGB
971  */
972 static inline void RENAME(yuv2rgb32_2)(SwsContext *c, const uint16_t *buf0,
973                                        const uint16_t *buf1, const uint16_t *ubuf0,
974                                        const uint16_t *ubuf1, const uint16_t *vbuf0,
975                                        const uint16_t *vbuf1, const uint16_t *abuf0,
976                                        const uint16_t *abuf1, uint8_t *dest,
977                                        int dstW, int yalpha, int uvalpha, int y)
978 {
979     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
980 #if ARCH_X86_64
981         __asm__ volatile(
982             YSCALEYUV2RGB(%%r8, %5)
983             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
984             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
985             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
986             "packuswb            %%mm7, %%mm1       \n\t"
987             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
988             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "r" (dest),
989                "a" (&c->redDither),
990                "r" (abuf0), "r" (abuf1)
991             : "%r8"
992         );
993 #else
994         *(const uint16_t **)(&c->u_temp)=abuf0;
995         *(const uint16_t **)(&c->v_temp)=abuf1;
996         __asm__ volatile(
997             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
998             "mov        %4, %%"REG_b"               \n\t"
999             "push %%"REG_BP"                        \n\t"
1000             YSCALEYUV2RGB(%%REGBP, %5)
1001             "push                   %0              \n\t"
1002             "push                   %1              \n\t"
1003             "mov          "U_TEMP"(%5), %0          \n\t"
1004             "mov          "V_TEMP"(%5), %1          \n\t"
1005             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
1006             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1007             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1008             "packuswb            %%mm7, %%mm1       \n\t"
1009             "pop                    %1              \n\t"
1010             "pop                    %0              \n\t"
1011             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1012             "pop %%"REG_BP"                         \n\t"
1013             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1014             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1015                "a" (&c->redDither)
1016         );
1017 #endif
1018     } else {
1019         __asm__ volatile(
1020             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1021             "mov        %4, %%"REG_b"               \n\t"
1022             "push %%"REG_BP"                        \n\t"
1023             YSCALEYUV2RGB(%%REGBP, %5)
1024             "pcmpeqd %%mm7, %%mm7                   \n\t"
1025             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1026             "pop %%"REG_BP"                         \n\t"
1027             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1028             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1029                "a" (&c->redDither)
1030         );
1031     }
1032 }
1033
1034 static inline void RENAME(yuv2bgr24_2)(SwsContext *c, const uint16_t *buf0,
1035                                        const uint16_t *buf1, const uint16_t *ubuf0,
1036                                        const uint16_t *ubuf1, const uint16_t *vbuf0,
1037                                        const uint16_t *vbuf1, const uint16_t *abuf0,
1038                                        const uint16_t *abuf1, uint8_t *dest,
1039                                        int dstW, int yalpha, int uvalpha, int y)
1040 {
1041     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1042     __asm__ volatile(
1043         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1044         "mov        %4, %%"REG_b"               \n\t"
1045         "push %%"REG_BP"                        \n\t"
1046         YSCALEYUV2RGB(%%REGBP, %5)
1047         "pxor    %%mm7, %%mm7                   \n\t"
1048         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1049         "pop %%"REG_BP"                         \n\t"
1050         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1051         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1052            "a" (&c->redDither)
1053     );
1054 }
1055
1056 static inline void RENAME(yuv2rgb555_2)(SwsContext *c, const uint16_t *buf0,
1057                                         const uint16_t *buf1, const uint16_t *ubuf0,
1058                                         const uint16_t *ubuf1, const uint16_t *vbuf0,
1059                                         const uint16_t *vbuf1, const uint16_t *abuf0,
1060                                         const uint16_t *abuf1, uint8_t *dest,
1061                                         int dstW, int yalpha, int uvalpha, int y)
1062 {
1063     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1064     __asm__ volatile(
1065         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1066         "mov        %4, %%"REG_b"               \n\t"
1067         "push %%"REG_BP"                        \n\t"
1068         YSCALEYUV2RGB(%%REGBP, %5)
1069         "pxor    %%mm7, %%mm7                   \n\t"
1070         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1071 #ifdef DITHER1XBPP
1072         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1073         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1074         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1075 #endif
1076         WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1077         "pop %%"REG_BP"                         \n\t"
1078         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1079         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1080            "a" (&c->redDither)
1081     );
1082 }
1083
1084 static inline void RENAME(yuv2rgb565_2)(SwsContext *c, const uint16_t *buf0,
1085                                         const uint16_t *buf1, const uint16_t *ubuf0,
1086                                         const uint16_t *ubuf1, const uint16_t *vbuf0,
1087                                         const uint16_t *vbuf1, const uint16_t *abuf0,
1088                                         const uint16_t *abuf1, uint8_t *dest,
1089                                         int dstW, int yalpha, int uvalpha, int y)
1090 {
1091     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1092     __asm__ volatile(
1093         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1094         "mov        %4, %%"REG_b"               \n\t"
1095         "push %%"REG_BP"                        \n\t"
1096         YSCALEYUV2RGB(%%REGBP, %5)
1097         "pxor    %%mm7, %%mm7                   \n\t"
1098         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1099 #ifdef DITHER1XBPP
1100         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1101         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1102         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1103 #endif
1104         WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1105         "pop %%"REG_BP"                         \n\t"
1106         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1107         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1108            "a" (&c->redDither)
1109     );
1110 }
1111
1112 #define REAL_YSCALEYUV2PACKED(index, c) \
1113     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
1114     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
1115     "psraw                $3, %%mm0                           \n\t"\
1116     "psraw                $3, %%mm1                           \n\t"\
1117     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1118     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1119     "xor            "#index", "#index"                        \n\t"\
1120     ".p2align              4            \n\t"\
1121     "1:                                 \n\t"\
1122     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1123     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1124     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1125     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1126     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1127     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1128     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
1129     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
1130     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
1131     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
1132     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
1133     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1134     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1135     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
1136     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
1137     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
1138     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
1139     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
1140     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
1141     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
1142     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
1143     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1144     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1145     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1146     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1147     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1148     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1149
1150 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
1151
1152 static inline void RENAME(yuv2yuyv422_2)(SwsContext *c, const uint16_t *buf0,
1153                                          const uint16_t *buf1, const uint16_t *ubuf0,
1154                                          const uint16_t *ubuf1, const uint16_t *vbuf0,
1155                                          const uint16_t *vbuf1, const uint16_t *abuf0,
1156                                          const uint16_t *abuf1, uint8_t *dest,
1157                                          int dstW, int yalpha, int uvalpha, int y)
1158 {
1159     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1160     __asm__ volatile(
1161         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1162         "mov %4, %%"REG_b"                        \n\t"
1163         "push %%"REG_BP"                        \n\t"
1164         YSCALEYUV2PACKED(%%REGBP, %5)
1165         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1166         "pop %%"REG_BP"                         \n\t"
1167         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1168         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1169            "a" (&c->redDither)
1170     );
1171 }
1172
1173 #define REAL_YSCALEYUV2RGB1(index, c) \
1174     "xor            "#index", "#index"  \n\t"\
1175     ".p2align              4            \n\t"\
1176     "1:                                 \n\t"\
1177     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1178     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1179     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1180     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1181     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1182     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1183     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1184     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1185     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1186     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1187     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1188     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1189     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1190     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1191     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1192     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1193     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1194     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1195     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1196     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1197     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1198     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1199     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1200     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1201     "paddw             %%mm3, %%mm4     \n\t"\
1202     "movq              %%mm2, %%mm0     \n\t"\
1203     "movq              %%mm5, %%mm6     \n\t"\
1204     "movq              %%mm4, %%mm3     \n\t"\
1205     "punpcklwd         %%mm2, %%mm2     \n\t"\
1206     "punpcklwd         %%mm5, %%mm5     \n\t"\
1207     "punpcklwd         %%mm4, %%mm4     \n\t"\
1208     "paddw             %%mm1, %%mm2     \n\t"\
1209     "paddw             %%mm1, %%mm5     \n\t"\
1210     "paddw             %%mm1, %%mm4     \n\t"\
1211     "punpckhwd         %%mm0, %%mm0     \n\t"\
1212     "punpckhwd         %%mm6, %%mm6     \n\t"\
1213     "punpckhwd         %%mm3, %%mm3     \n\t"\
1214     "paddw             %%mm7, %%mm0     \n\t"\
1215     "paddw             %%mm7, %%mm6     \n\t"\
1216     "paddw             %%mm7, %%mm3     \n\t"\
1217     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1218     "packuswb          %%mm0, %%mm2     \n\t"\
1219     "packuswb          %%mm6, %%mm5     \n\t"\
1220     "packuswb          %%mm3, %%mm4     \n\t"\
1221
1222 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
1223
1224 // do vertical chrominance interpolation
1225 #define REAL_YSCALEYUV2RGB1b(index, c) \
1226     "xor            "#index", "#index"  \n\t"\
1227     ".p2align              4            \n\t"\
1228     "1:                                 \n\t"\
1229     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1230     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1231     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1232     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1233     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1234     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1235     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1236     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1237     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
1238     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
1239     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1240     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1241     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1242     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1243     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1244     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1245     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1246     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1247     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1248     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1249     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1250     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1251     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1252     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1253     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1254     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1255     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1256     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1257     "paddw             %%mm3, %%mm4     \n\t"\
1258     "movq              %%mm2, %%mm0     \n\t"\
1259     "movq              %%mm5, %%mm6     \n\t"\
1260     "movq              %%mm4, %%mm3     \n\t"\
1261     "punpcklwd         %%mm2, %%mm2     \n\t"\
1262     "punpcklwd         %%mm5, %%mm5     \n\t"\
1263     "punpcklwd         %%mm4, %%mm4     \n\t"\
1264     "paddw             %%mm1, %%mm2     \n\t"\
1265     "paddw             %%mm1, %%mm5     \n\t"\
1266     "paddw             %%mm1, %%mm4     \n\t"\
1267     "punpckhwd         %%mm0, %%mm0     \n\t"\
1268     "punpckhwd         %%mm6, %%mm6     \n\t"\
1269     "punpckhwd         %%mm3, %%mm3     \n\t"\
1270     "paddw             %%mm7, %%mm0     \n\t"\
1271     "paddw             %%mm7, %%mm6     \n\t"\
1272     "paddw             %%mm7, %%mm3     \n\t"\
1273     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1274     "packuswb          %%mm0, %%mm2     \n\t"\
1275     "packuswb          %%mm6, %%mm5     \n\t"\
1276     "packuswb          %%mm3, %%mm4     \n\t"\
1277
1278 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
1279
1280 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
1281     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
1282     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
1283     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
1284     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
1285     "packuswb          %%mm1, %%mm7     \n\t"
1286 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
1287
1288 /**
1289  * YV12 to RGB without scaling or interpolating
1290  */
1291 static inline void RENAME(yuv2rgb32_1)(SwsContext *c, const uint16_t *buf0,
1292                                        const uint16_t *ubuf0, const uint16_t *ubuf1,
1293                                        const uint16_t *vbuf0, const uint16_t *vbuf1,
1294                                        const uint16_t *abuf0, uint8_t *dest,
1295                                        int dstW, int uvalpha, enum PixelFormat dstFormat,
1296                                        int flags, int y)
1297 {
1298     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1299
1300     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1301         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1302             __asm__ volatile(
1303                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1304                 "mov        %4, %%"REG_b"               \n\t"
1305                 "push %%"REG_BP"                        \n\t"
1306                 YSCALEYUV2RGB1(%%REGBP, %5)
1307                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1308                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1309                 "pop %%"REG_BP"                         \n\t"
1310                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1311                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1312                    "a" (&c->redDither)
1313             );
1314         } else {
1315             __asm__ volatile(
1316                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1317                 "mov        %4, %%"REG_b"               \n\t"
1318                 "push %%"REG_BP"                        \n\t"
1319                 YSCALEYUV2RGB1(%%REGBP, %5)
1320                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1321                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1322                 "pop %%"REG_BP"                         \n\t"
1323                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1324                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1325                    "a" (&c->redDither)
1326             );
1327         }
1328     } else {
1329         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1330             __asm__ volatile(
1331                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1332                 "mov        %4, %%"REG_b"               \n\t"
1333                 "push %%"REG_BP"                        \n\t"
1334                 YSCALEYUV2RGB1b(%%REGBP, %5)
1335                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1336                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1337                 "pop %%"REG_BP"                         \n\t"
1338                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1339                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1340                    "a" (&c->redDither)
1341             );
1342         } else {
1343             __asm__ volatile(
1344                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1345                 "mov        %4, %%"REG_b"               \n\t"
1346                 "push %%"REG_BP"                        \n\t"
1347                 YSCALEYUV2RGB1b(%%REGBP, %5)
1348                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1349                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1350                 "pop %%"REG_BP"                         \n\t"
1351                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1352                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1353                    "a" (&c->redDither)
1354             );
1355         }
1356     }
1357 }
1358
1359 static inline void RENAME(yuv2bgr24_1)(SwsContext *c, const uint16_t *buf0,
1360                                        const uint16_t *ubuf0, const uint16_t *ubuf1,
1361                                        const uint16_t *vbuf0, const uint16_t *vbuf1,
1362                                        const uint16_t *abuf0, uint8_t *dest,
1363                                        int dstW, int uvalpha, enum PixelFormat dstFormat,
1364                                        int flags, int y)
1365 {
1366     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1367
1368     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1369         __asm__ volatile(
1370             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1371             "mov        %4, %%"REG_b"               \n\t"
1372             "push %%"REG_BP"                        \n\t"
1373             YSCALEYUV2RGB1(%%REGBP, %5)
1374             "pxor    %%mm7, %%mm7                   \n\t"
1375             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1376             "pop %%"REG_BP"                         \n\t"
1377             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1378             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1379                "a" (&c->redDither)
1380         );
1381     } else {
1382         __asm__ volatile(
1383             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1384             "mov        %4, %%"REG_b"               \n\t"
1385             "push %%"REG_BP"                        \n\t"
1386             YSCALEYUV2RGB1b(%%REGBP, %5)
1387             "pxor    %%mm7, %%mm7                   \n\t"
1388             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1389             "pop %%"REG_BP"                         \n\t"
1390             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1391             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1392                "a" (&c->redDither)
1393         );
1394     }
1395 }
1396
1397 static inline void RENAME(yuv2rgb555_1)(SwsContext *c, const uint16_t *buf0,
1398                                         const uint16_t *ubuf0, const uint16_t *ubuf1,
1399                                         const uint16_t *vbuf0, const uint16_t *vbuf1,
1400                                         const uint16_t *abuf0, uint8_t *dest,
1401                                         int dstW, int uvalpha, enum PixelFormat dstFormat,
1402                                         int flags, int y)
1403 {
1404     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1405
1406     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1407         __asm__ volatile(
1408             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1409             "mov        %4, %%"REG_b"               \n\t"
1410             "push %%"REG_BP"                        \n\t"
1411             YSCALEYUV2RGB1(%%REGBP, %5)
1412             "pxor    %%mm7, %%mm7                   \n\t"
1413             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1414 #ifdef DITHER1XBPP
1415             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1416             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1417             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1418 #endif
1419             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1420             "pop %%"REG_BP"                         \n\t"
1421             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1422             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1423                "a" (&c->redDither)
1424         );
1425     } else {
1426         __asm__ volatile(
1427             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1428             "mov        %4, %%"REG_b"               \n\t"
1429             "push %%"REG_BP"                        \n\t"
1430             YSCALEYUV2RGB1b(%%REGBP, %5)
1431             "pxor    %%mm7, %%mm7                   \n\t"
1432             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1433 #ifdef DITHER1XBPP
1434             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1435             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1436             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1437 #endif
1438             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1439             "pop %%"REG_BP"                         \n\t"
1440             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1441             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1442                "a" (&c->redDither)
1443         );
1444     }
1445 }
1446
1447 static inline void RENAME(yuv2rgb565_1)(SwsContext *c, const uint16_t *buf0,
1448                                         const uint16_t *ubuf0, const uint16_t *ubuf1,
1449                                         const uint16_t *vbuf0, const uint16_t *vbuf1,
1450                                         const uint16_t *abuf0, uint8_t *dest,
1451                                         int dstW, int uvalpha, enum PixelFormat dstFormat,
1452                                         int flags, int y)
1453 {
1454     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1455
1456     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1457         __asm__ volatile(
1458             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1459             "mov        %4, %%"REG_b"               \n\t"
1460             "push %%"REG_BP"                        \n\t"
1461             YSCALEYUV2RGB1(%%REGBP, %5)
1462             "pxor    %%mm7, %%mm7                   \n\t"
1463             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1464 #ifdef DITHER1XBPP
1465             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1466             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1467             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1468 #endif
1469             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1470             "pop %%"REG_BP"                         \n\t"
1471             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1472             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1473                "a" (&c->redDither)
1474         );
1475     } else {
1476         __asm__ volatile(
1477             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1478             "mov        %4, %%"REG_b"               \n\t"
1479             "push %%"REG_BP"                        \n\t"
1480             YSCALEYUV2RGB1b(%%REGBP, %5)
1481             "pxor    %%mm7, %%mm7                   \n\t"
1482             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1483 #ifdef DITHER1XBPP
1484             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1485             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1486             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1487 #endif
1488             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1489             "pop %%"REG_BP"                         \n\t"
1490             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1491             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1492                "a" (&c->redDither)
1493         );
1494     }
1495 }
1496
1497 #define REAL_YSCALEYUV2PACKED1(index, c) \
1498     "xor            "#index", "#index"  \n\t"\
1499     ".p2align              4            \n\t"\
1500     "1:                                 \n\t"\
1501     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1502     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1503     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1504     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1505     "psraw                $7, %%mm3     \n\t" \
1506     "psraw                $7, %%mm4     \n\t" \
1507     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1508     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1509     "psraw                $7, %%mm1     \n\t" \
1510     "psraw                $7, %%mm7     \n\t" \
1511
1512 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
1513
1514 #define REAL_YSCALEYUV2PACKED1b(index, c) \
1515     "xor "#index", "#index"             \n\t"\
1516     ".p2align              4            \n\t"\
1517     "1:                                 \n\t"\
1518     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1519     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1520     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1521     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1522     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1523     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1524     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1525     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1526     "psrlw                $8, %%mm3     \n\t" \
1527     "psrlw                $8, %%mm4     \n\t" \
1528     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1529     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1530     "psraw                $7, %%mm1     \n\t" \
1531     "psraw                $7, %%mm7     \n\t"
1532 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
1533
1534 static inline void RENAME(yuv2yuyv422_1)(SwsContext *c, const uint16_t *buf0,
1535                                          const uint16_t *ubuf0, const uint16_t *ubuf1,
1536                                          const uint16_t *vbuf0, const uint16_t *vbuf1,
1537                                          const uint16_t *abuf0, uint8_t *dest,
1538                                          int dstW, int uvalpha, enum PixelFormat dstFormat,
1539                                          int flags, int y)
1540 {
1541     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1542
1543     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1544         __asm__ volatile(
1545             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1546             "mov        %4, %%"REG_b"               \n\t"
1547             "push %%"REG_BP"                        \n\t"
1548             YSCALEYUV2PACKED1(%%REGBP, %5)
1549             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1550             "pop %%"REG_BP"                         \n\t"
1551             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1552             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1553                "a" (&c->redDither)
1554         );
1555     } else {
1556         __asm__ volatile(
1557             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1558             "mov        %4, %%"REG_b"               \n\t"
1559             "push %%"REG_BP"                        \n\t"
1560             YSCALEYUV2PACKED1b(%%REGBP, %5)
1561             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1562             "pop %%"REG_BP"                         \n\t"
1563             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1564             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1565                "a" (&c->redDither)
1566         );
1567     }
1568 }
1569
1570 #if !COMPILE_TEMPLATE_MMX2
1571 //FIXME yuy2* can read up to 7 samples too much
1572
1573 static inline void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src, int width, uint32_t *unused)
1574 {
1575     __asm__ volatile(
1576         "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1577         "mov                    %0, %%"REG_a"       \n\t"
1578         "1:                                         \n\t"
1579         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1580         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1581         "pand                %%mm2, %%mm0           \n\t"
1582         "pand                %%mm2, %%mm1           \n\t"
1583         "packuswb            %%mm1, %%mm0           \n\t"
1584         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1585         "add                    $8, %%"REG_a"       \n\t"
1586         " js                    1b                  \n\t"
1587         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1588         : "%"REG_a
1589     );
1590 }
1591
1592 static inline void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)
1593 {
1594     __asm__ volatile(
1595         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1596         "mov                    %0, %%"REG_a"       \n\t"
1597         "1:                                         \n\t"
1598         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1599         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1600         "psrlw                  $8, %%mm0           \n\t"
1601         "psrlw                  $8, %%mm1           \n\t"
1602         "packuswb            %%mm1, %%mm0           \n\t"
1603         "movq                %%mm0, %%mm1           \n\t"
1604         "psrlw                  $8, %%mm0           \n\t"
1605         "pand                %%mm4, %%mm1           \n\t"
1606         "packuswb            %%mm0, %%mm0           \n\t"
1607         "packuswb            %%mm1, %%mm1           \n\t"
1608         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1609         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1610         "add                    $4, %%"REG_a"       \n\t"
1611         " js                    1b                  \n\t"
1612         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1613         : "%"REG_a
1614     );
1615     assert(src1 == src2);
1616 }
1617
1618 static inline void RENAME(LEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)
1619 {
1620     __asm__ volatile(
1621         "mov                    %0, %%"REG_a"       \n\t"
1622         "1:                                         \n\t"
1623         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1624         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1625         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1626         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1627         "psrlw                  $8, %%mm0           \n\t"
1628         "psrlw                  $8, %%mm1           \n\t"
1629         "psrlw                  $8, %%mm2           \n\t"
1630         "psrlw                  $8, %%mm3           \n\t"
1631         "packuswb            %%mm1, %%mm0           \n\t"
1632         "packuswb            %%mm3, %%mm2           \n\t"
1633         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1634         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1635         "add                    $8, %%"REG_a"       \n\t"
1636         " js                    1b                  \n\t"
1637         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1638         : "%"REG_a
1639     );
1640 }
1641
1642 /* This is almost identical to the previous, end exists only because
1643  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1644 static inline void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src, int width, uint32_t *unused)
1645 {
1646     __asm__ volatile(
1647         "mov                  %0, %%"REG_a"         \n\t"
1648         "1:                                         \n\t"
1649         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1650         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1651         "psrlw                $8, %%mm0             \n\t"
1652         "psrlw                $8, %%mm1             \n\t"
1653         "packuswb          %%mm1, %%mm0             \n\t"
1654         "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1655         "add                  $8, %%"REG_a"         \n\t"
1656         " js                  1b                    \n\t"
1657         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1658         : "%"REG_a
1659     );
1660 }
1661
1662 static inline void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)
1663 {
1664     __asm__ volatile(
1665         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1666         "mov                    %0, %%"REG_a"       \n\t"
1667         "1:                                         \n\t"
1668         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1669         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1670         "pand                %%mm4, %%mm0           \n\t"
1671         "pand                %%mm4, %%mm1           \n\t"
1672         "packuswb            %%mm1, %%mm0           \n\t"
1673         "movq                %%mm0, %%mm1           \n\t"
1674         "psrlw                  $8, %%mm0           \n\t"
1675         "pand                %%mm4, %%mm1           \n\t"
1676         "packuswb            %%mm0, %%mm0           \n\t"
1677         "packuswb            %%mm1, %%mm1           \n\t"
1678         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1679         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1680         "add                    $4, %%"REG_a"       \n\t"
1681         " js                    1b                  \n\t"
1682         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1683         : "%"REG_a
1684     );
1685     assert(src1 == src2);
1686 }
1687
1688 static inline void RENAME(BEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)
1689 {
1690     __asm__ volatile(
1691         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1692         "mov                    %0, %%"REG_a"       \n\t"
1693         "1:                                         \n\t"
1694         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1695         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1696         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1697         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1698         "pand                %%mm4, %%mm0           \n\t"
1699         "pand                %%mm4, %%mm1           \n\t"
1700         "pand                %%mm4, %%mm2           \n\t"
1701         "pand                %%mm4, %%mm3           \n\t"
1702         "packuswb            %%mm1, %%mm0           \n\t"
1703         "packuswb            %%mm3, %%mm2           \n\t"
1704         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1705         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1706         "add                    $8, %%"REG_a"       \n\t"
1707         " js                    1b                  \n\t"
1708         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1709         : "%"REG_a
1710     );
1711 }
1712
1713 static inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1714                                     const uint8_t *src, int width)
1715 {
1716     __asm__ volatile(
1717         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1718         "mov                    %0, %%"REG_a"       \n\t"
1719         "1:                                         \n\t"
1720         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1721         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1722         "movq                %%mm0, %%mm2           \n\t"
1723         "movq                %%mm1, %%mm3           \n\t"
1724         "pand                %%mm4, %%mm0           \n\t"
1725         "pand                %%mm4, %%mm1           \n\t"
1726         "psrlw                  $8, %%mm2           \n\t"
1727         "psrlw                  $8, %%mm3           \n\t"
1728         "packuswb            %%mm1, %%mm0           \n\t"
1729         "packuswb            %%mm3, %%mm2           \n\t"
1730         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1731         "movq                %%mm2, (%3, %%"REG_a") \n\t"
1732         "add                    $8, %%"REG_a"       \n\t"
1733         " js                    1b                  \n\t"
1734         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1735         : "%"REG_a
1736     );
1737 }
1738
1739 static inline void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1740                                     const uint8_t *src1, const uint8_t *src2,
1741                                     int width, uint32_t *unused)
1742 {
1743     RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1744 }
1745
1746 static inline void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1747                                     const uint8_t *src1, const uint8_t *src2,
1748                                     int width, uint32_t *unused)
1749 {
1750     RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1751 }
1752 #endif /* !COMPILE_TEMPLATE_MMX2 */
1753
1754 static inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, const uint8_t *src, int width, enum PixelFormat srcFormat)
1755 {
1756
1757     if(srcFormat == PIX_FMT_BGR24) {
1758         __asm__ volatile(
1759             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1760             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1761             :
1762         );
1763     } else {
1764         __asm__ volatile(
1765             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1766             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1767             :
1768         );
1769     }
1770
1771     __asm__ volatile(
1772         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1773         "mov                        %2, %%"REG_a"   \n\t"
1774         "pxor                    %%mm7, %%mm7       \n\t"
1775         "1:                                         \n\t"
1776         PREFETCH"               64(%0)              \n\t"
1777         "movd                     (%0), %%mm0       \n\t"
1778         "movd                    2(%0), %%mm1       \n\t"
1779         "movd                    6(%0), %%mm2       \n\t"
1780         "movd                    8(%0), %%mm3       \n\t"
1781         "add                       $12, %0          \n\t"
1782         "punpcklbw               %%mm7, %%mm0       \n\t"
1783         "punpcklbw               %%mm7, %%mm1       \n\t"
1784         "punpcklbw               %%mm7, %%mm2       \n\t"
1785         "punpcklbw               %%mm7, %%mm3       \n\t"
1786         "pmaddwd                 %%mm5, %%mm0       \n\t"
1787         "pmaddwd                 %%mm6, %%mm1       \n\t"
1788         "pmaddwd                 %%mm5, %%mm2       \n\t"
1789         "pmaddwd                 %%mm6, %%mm3       \n\t"
1790         "paddd                   %%mm1, %%mm0       \n\t"
1791         "paddd                   %%mm3, %%mm2       \n\t"
1792         "paddd                   %%mm4, %%mm0       \n\t"
1793         "paddd                   %%mm4, %%mm2       \n\t"
1794         "psrad                     $15, %%mm0       \n\t"
1795         "psrad                     $15, %%mm2       \n\t"
1796         "packssdw                %%mm2, %%mm0       \n\t"
1797         "packuswb                %%mm0, %%mm0       \n\t"
1798         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1799         "add                        $4, %%"REG_a"   \n\t"
1800         " js                        1b              \n\t"
1801     : "+r" (src)
1802     : "r" (dst+width), "g" ((x86_reg)-width)
1803     : "%"REG_a
1804     );
1805 }
1806
1807 static inline void RENAME(bgr24ToUV_mmx)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src, int width, enum PixelFormat srcFormat)
1808 {
1809     __asm__ volatile(
1810         "movq                    24(%4), %%mm6       \n\t"
1811         "mov                        %3, %%"REG_a"   \n\t"
1812         "pxor                    %%mm7, %%mm7       \n\t"
1813         "1:                                         \n\t"
1814         PREFETCH"               64(%0)              \n\t"
1815         "movd                     (%0), %%mm0       \n\t"
1816         "movd                    2(%0), %%mm1       \n\t"
1817         "punpcklbw               %%mm7, %%mm0       \n\t"
1818         "punpcklbw               %%mm7, %%mm1       \n\t"
1819         "movq                    %%mm0, %%mm2       \n\t"
1820         "movq                    %%mm1, %%mm3       \n\t"
1821         "pmaddwd                  (%4), %%mm0       \n\t"
1822         "pmaddwd                 8(%4), %%mm1       \n\t"
1823         "pmaddwd                16(%4), %%mm2       \n\t"
1824         "pmaddwd                 %%mm6, %%mm3       \n\t"
1825         "paddd                   %%mm1, %%mm0       \n\t"
1826         "paddd                   %%mm3, %%mm2       \n\t"
1827
1828         "movd                    6(%0), %%mm1       \n\t"
1829         "movd                    8(%0), %%mm3       \n\t"
1830         "add                       $12, %0          \n\t"
1831         "punpcklbw               %%mm7, %%mm1       \n\t"
1832         "punpcklbw               %%mm7, %%mm3       \n\t"
1833         "movq                    %%mm1, %%mm4       \n\t"
1834         "movq                    %%mm3, %%mm5       \n\t"
1835         "pmaddwd                  (%4), %%mm1       \n\t"
1836         "pmaddwd                 8(%4), %%mm3       \n\t"
1837         "pmaddwd                16(%4), %%mm4       \n\t"
1838         "pmaddwd                 %%mm6, %%mm5       \n\t"
1839         "paddd                   %%mm3, %%mm1       \n\t"
1840         "paddd                   %%mm5, %%mm4       \n\t"
1841
1842         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1843         "paddd                   %%mm3, %%mm0       \n\t"
1844         "paddd                   %%mm3, %%mm2       \n\t"
1845         "paddd                   %%mm3, %%mm1       \n\t"
1846         "paddd                   %%mm3, %%mm4       \n\t"
1847         "psrad                     $15, %%mm0       \n\t"
1848         "psrad                     $15, %%mm2       \n\t"
1849         "psrad                     $15, %%mm1       \n\t"
1850         "psrad                     $15, %%mm4       \n\t"
1851         "packssdw                %%mm1, %%mm0       \n\t"
1852         "packssdw                %%mm4, %%mm2       \n\t"
1853         "packuswb                %%mm0, %%mm0       \n\t"
1854         "packuswb                %%mm2, %%mm2       \n\t"
1855         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1856         "movd                %%mm2, (%2, %%"REG_a") \n\t"
1857         "add                        $4, %%"REG_a"   \n\t"
1858         " js                        1b              \n\t"
1859     : "+r" (src)
1860     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1861     : "%"REG_a
1862     );
1863 }
1864
1865 static inline void RENAME(bgr24ToY)(uint8_t *dst, const uint8_t *src, int width, uint32_t *unused)
1866 {
1867     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1868 }
1869
1870 static inline void RENAME(bgr24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)
1871 {
1872     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1873     assert(src1 == src2);
1874 }
1875
1876 static inline void RENAME(rgb24ToY)(uint8_t *dst, const uint8_t *src, int width, uint32_t *unused)
1877 {
1878     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1879 }
1880
1881 static inline void RENAME(rgb24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, int width, uint32_t *unused)
1882 {
1883     assert(src1==src2);
1884     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1885 }
1886
1887 #if !COMPILE_TEMPLATE_MMX2
1888 // bilinear / bicubic scaling
1889 static inline void RENAME(hScale)(int16_t *dst, int dstW, const uint8_t *src, int srcW, int xInc,
1890                                   const int16_t *filter, const int16_t *filterPos, int filterSize)
1891 {
1892     assert(filterSize % 4 == 0 && filterSize>0);
1893     if (filterSize==4) { // Always true for upscaling, sometimes for down, too.
1894         x86_reg counter= -2*dstW;
1895         filter-= counter*2;
1896         filterPos-= counter/2;
1897         dst-= counter/2;
1898         __asm__ volatile(
1899 #if defined(PIC)
1900             "push            %%"REG_b"              \n\t"
1901 #endif
1902             "pxor                %%mm7, %%mm7       \n\t"
1903             "push           %%"REG_BP"              \n\t" // we use 7 regs here ...
1904             "mov             %%"REG_a", %%"REG_BP"  \n\t"
1905             ".p2align                4              \n\t"
1906             "1:                                     \n\t"
1907             "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"
1908             "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"
1909             "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"
1910             "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"
1911             "movd      (%3, %%"REG_a"), %%mm0       \n\t"
1912             "movd      (%3, %%"REG_b"), %%mm2       \n\t"
1913             "punpcklbw           %%mm7, %%mm0       \n\t"
1914             "punpcklbw           %%mm7, %%mm2       \n\t"
1915             "pmaddwd             %%mm1, %%mm0       \n\t"
1916             "pmaddwd             %%mm2, %%mm3       \n\t"
1917             "movq                %%mm0, %%mm4       \n\t"
1918             "punpckldq           %%mm3, %%mm0       \n\t"
1919             "punpckhdq           %%mm3, %%mm4       \n\t"
1920             "paddd               %%mm4, %%mm0       \n\t"
1921             "psrad                  $7, %%mm0       \n\t"
1922             "packssdw            %%mm0, %%mm0       \n\t"
1923             "movd                %%mm0, (%4, %%"REG_BP")    \n\t"
1924             "add                    $4, %%"REG_BP"  \n\t"
1925             " jnc                   1b              \n\t"
1926
1927             "pop            %%"REG_BP"              \n\t"
1928 #if defined(PIC)
1929             "pop             %%"REG_b"              \n\t"
1930 #endif
1931             : "+a" (counter)
1932             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1933 #if !defined(PIC)
1934             : "%"REG_b
1935 #endif
1936         );
1937     } else if (filterSize==8) {
1938         x86_reg counter= -2*dstW;
1939         filter-= counter*4;
1940         filterPos-= counter/2;
1941         dst-= counter/2;
1942         __asm__ volatile(
1943 #if defined(PIC)
1944             "push             %%"REG_b"             \n\t"
1945 #endif
1946             "pxor                 %%mm7, %%mm7      \n\t"
1947             "push            %%"REG_BP"             \n\t" // we use 7 regs here ...
1948             "mov              %%"REG_a", %%"REG_BP" \n\t"
1949             ".p2align                 4             \n\t"
1950             "1:                                     \n\t"
1951             "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"
1952             "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"
1953             "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"
1954             "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"
1955             "movd       (%3, %%"REG_a"), %%mm0      \n\t"
1956             "movd       (%3, %%"REG_b"), %%mm2      \n\t"
1957             "punpcklbw            %%mm7, %%mm0      \n\t"
1958             "punpcklbw            %%mm7, %%mm2      \n\t"
1959             "pmaddwd              %%mm1, %%mm0      \n\t"
1960             "pmaddwd              %%mm2, %%mm3      \n\t"
1961
1962             "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"
1963             "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"
1964             "movd      4(%3, %%"REG_a"), %%mm4      \n\t"
1965             "movd      4(%3, %%"REG_b"), %%mm2      \n\t"
1966             "punpcklbw            %%mm7, %%mm4      \n\t"
1967             "punpcklbw            %%mm7, %%mm2      \n\t"
1968             "pmaddwd              %%mm1, %%mm4      \n\t"
1969             "pmaddwd              %%mm2, %%mm5      \n\t"
1970             "paddd                %%mm4, %%mm0      \n\t"
1971             "paddd                %%mm5, %%mm3      \n\t"
1972             "movq                 %%mm0, %%mm4      \n\t"
1973             "punpckldq            %%mm3, %%mm0      \n\t"
1974             "punpckhdq            %%mm3, %%mm4      \n\t"
1975             "paddd                %%mm4, %%mm0      \n\t"
1976             "psrad                   $7, %%mm0      \n\t"
1977             "packssdw             %%mm0, %%mm0      \n\t"
1978             "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"
1979             "add                     $4, %%"REG_BP" \n\t"
1980             " jnc                    1b             \n\t"
1981
1982             "pop             %%"REG_BP"             \n\t"
1983 #if defined(PIC)
1984             "pop              %%"REG_b"             \n\t"
1985 #endif
1986             : "+a" (counter)
1987             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1988 #if !defined(PIC)
1989             : "%"REG_b
1990 #endif
1991         );
1992     } else {
1993         const uint8_t *offset = src+filterSize;
1994         x86_reg counter= -2*dstW;
1995         //filter-= counter*filterSize/2;
1996         filterPos-= counter/2;
1997         dst-= counter/2;
1998         __asm__ volatile(
1999             "pxor                  %%mm7, %%mm7     \n\t"
2000             ".p2align                  4            \n\t"
2001             "1:                                     \n\t"
2002             "mov                      %2, %%"REG_c" \n\t"
2003             "movzwl      (%%"REG_c", %0), %%eax     \n\t"
2004             "movzwl     2(%%"REG_c", %0), %%edx     \n\t"
2005             "mov                      %5, %%"REG_c" \n\t"
2006             "pxor                  %%mm4, %%mm4     \n\t"
2007             "pxor                  %%mm5, %%mm5     \n\t"
2008             "2:                                     \n\t"
2009             "movq                   (%1), %%mm1     \n\t"
2010             "movq               (%1, %6), %%mm3     \n\t"
2011             "movd (%%"REG_c", %%"REG_a"), %%mm0     \n\t"
2012             "movd (%%"REG_c", %%"REG_d"), %%mm2     \n\t"
2013             "punpcklbw             %%mm7, %%mm0     \n\t"
2014             "punpcklbw             %%mm7, %%mm2     \n\t"
2015             "pmaddwd               %%mm1, %%mm0     \n\t"
2016             "pmaddwd               %%mm2, %%mm3     \n\t"
2017             "paddd                 %%mm3, %%mm5     \n\t"
2018             "paddd                 %%mm0, %%mm4     \n\t"
2019             "add                      $8, %1        \n\t"
2020             "add                      $4, %%"REG_c" \n\t"
2021             "cmp                      %4, %%"REG_c" \n\t"
2022             " jb                      2b            \n\t"
2023             "add                      %6, %1        \n\t"
2024             "movq                  %%mm4, %%mm0     \n\t"
2025             "punpckldq             %%mm5, %%mm4     \n\t"
2026             "punpckhdq             %%mm5, %%mm0     \n\t"
2027             "paddd                 %%mm0, %%mm4     \n\t"
2028             "psrad                    $7, %%mm4     \n\t"
2029             "packssdw              %%mm4, %%mm4     \n\t"
2030             "mov                      %3, %%"REG_a" \n\t"
2031             "movd                  %%mm4, (%%"REG_a", %0)   \n\t"
2032             "add                      $4, %0        \n\t"
2033             " jnc                     1b            \n\t"
2034
2035             : "+r" (counter), "+r" (filter)
2036             : "m" (filterPos), "m" (dst), "m"(offset),
2037             "m" (src), "r" ((x86_reg)filterSize*2)
2038             : "%"REG_a, "%"REG_c, "%"REG_d
2039         );
2040     }
2041 }
2042 #endif /* !COMPILE_TEMPLATE_MMX2 */
2043
2044 #if COMPILE_TEMPLATE_MMX2
2045 static inline void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
2046                                         int dstWidth, const uint8_t *src, int srcW,
2047                                         int xInc)
2048 {
2049     int16_t *filterPos = c->hLumFilterPos;
2050     int16_t *filter    = c->hLumFilter;
2051     void    *mmx2FilterCode= c->lumMmx2FilterCode;
2052     int i;
2053 #if defined(PIC)
2054     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2055 #endif
2056
2057     __asm__ volatile(
2058 #if defined(PIC)
2059         "mov               %%"REG_b", %5        \n\t"
2060 #endif
2061         "pxor                  %%mm7, %%mm7     \n\t"
2062         "mov                      %0, %%"REG_c" \n\t"
2063         "mov                      %1, %%"REG_D" \n\t"
2064         "mov                      %2, %%"REG_d" \n\t"
2065         "mov                      %3, %%"REG_b" \n\t"
2066         "xor               %%"REG_a", %%"REG_a" \n\t" // i
2067         PREFETCH"        (%%"REG_c")            \n\t"
2068         PREFETCH"      32(%%"REG_c")            \n\t"
2069         PREFETCH"      64(%%"REG_c")            \n\t"
2070
2071 #if ARCH_X86_64
2072 #define CALL_MMX2_FILTER_CODE \
2073         "movl            (%%"REG_b"), %%esi     \n\t"\
2074         "call                    *%4            \n\t"\
2075         "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
2076         "add               %%"REG_S", %%"REG_c" \n\t"\
2077         "add               %%"REG_a", %%"REG_D" \n\t"\
2078         "xor               %%"REG_a", %%"REG_a" \n\t"\
2079
2080 #else
2081 #define CALL_MMX2_FILTER_CODE \
2082         "movl (%%"REG_b"), %%esi        \n\t"\
2083         "call         *%4                       \n\t"\
2084         "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
2085         "add               %%"REG_a", %%"REG_D" \n\t"\
2086         "xor               %%"REG_a", %%"REG_a" \n\t"\
2087
2088 #endif /* ARCH_X86_64 */
2089
2090         CALL_MMX2_FILTER_CODE
2091         CALL_MMX2_FILTER_CODE
2092         CALL_MMX2_FILTER_CODE
2093         CALL_MMX2_FILTER_CODE
2094         CALL_MMX2_FILTER_CODE
2095         CALL_MMX2_FILTER_CODE
2096         CALL_MMX2_FILTER_CODE
2097         CALL_MMX2_FILTER_CODE
2098
2099 #if defined(PIC)
2100         "mov                      %5, %%"REG_b" \n\t"
2101 #endif
2102         :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
2103            "m" (mmx2FilterCode)
2104 #if defined(PIC)
2105           ,"m" (ebxsave)
2106 #endif
2107         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2108 #if !defined(PIC)
2109          ,"%"REG_b
2110 #endif
2111     );
2112
2113     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
2114         dst[i] = src[srcW-1]*128;
2115 }
2116
2117 static inline void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2,
2118                                         int dstWidth, const uint8_t *src1,
2119                                         const uint8_t *src2, int srcW, int xInc)
2120 {
2121     int16_t *filterPos = c->hChrFilterPos;
2122     int16_t *filter    = c->hChrFilter;
2123     void    *mmx2FilterCode= c->chrMmx2FilterCode;
2124     int i;
2125 #if defined(PIC)
2126     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2127 #endif
2128
2129     __asm__ volatile(
2130 #if defined(PIC)
2131         "mov          %%"REG_b", %7         \n\t"
2132 #endif
2133         "pxor             %%mm7, %%mm7      \n\t"
2134         "mov                 %0, %%"REG_c"  \n\t"
2135         "mov                 %1, %%"REG_D"  \n\t"
2136         "mov                 %2, %%"REG_d"  \n\t"
2137         "mov                 %3, %%"REG_b"  \n\t"
2138         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2139         PREFETCH"   (%%"REG_c")             \n\t"
2140         PREFETCH" 32(%%"REG_c")             \n\t"
2141         PREFETCH" 64(%%"REG_c")             \n\t"
2142
2143         CALL_MMX2_FILTER_CODE
2144         CALL_MMX2_FILTER_CODE
2145         CALL_MMX2_FILTER_CODE
2146         CALL_MMX2_FILTER_CODE
2147         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2148         "mov                 %5, %%"REG_c"  \n\t" // src
2149         "mov                 %6, %%"REG_D"  \n\t" // buf2
2150         PREFETCH"   (%%"REG_c")             \n\t"
2151         PREFETCH" 32(%%"REG_c")             \n\t"
2152         PREFETCH" 64(%%"REG_c")             \n\t"
2153
2154         CALL_MMX2_FILTER_CODE
2155         CALL_MMX2_FILTER_CODE
2156         CALL_MMX2_FILTER_CODE
2157         CALL_MMX2_FILTER_CODE
2158
2159 #if defined(PIC)
2160         "mov %7, %%"REG_b"    \n\t"
2161 #endif
2162         :: "m" (src1), "m" (dst1), "m" (filter), "m" (filterPos),
2163            "m" (mmx2FilterCode), "m" (src2), "m"(dst2)
2164 #if defined(PIC)
2165           ,"m" (ebxsave)
2166 #endif
2167         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2168 #if !defined(PIC)
2169          ,"%"REG_b
2170 #endif
2171     );
2172
2173     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
2174         dst1[i] = src1[srcW-1]*128;
2175         dst2[i] = src2[srcW-1]*128;
2176     }
2177 }
2178 #endif /* COMPILE_TEMPLATE_MMX2 */
2179
2180 static void RENAME(sws_init_swScale)(SwsContext *c)
2181 {
2182     enum PixelFormat srcFormat = c->srcFormat,
2183                      dstFormat = c->dstFormat;
2184
2185     if (!is16BPS(dstFormat) && !is9_OR_10BPS(dstFormat)) {
2186         if (!(c->flags & SWS_BITEXACT)) {
2187             if (c->flags & SWS_ACCURATE_RND) {
2188                 c->yuv2yuv1 = RENAME(yuv2yuv1_ar    );
2189                 c->yuv2yuvX = RENAME(yuv2yuvX_ar    );
2190                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
2191                     switch (c->dstFormat) {
2192                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;
2193                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;
2194                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;
2195                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;
2196                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;
2197                     default: break;
2198                     }
2199                 }
2200             } else {
2201                 c->yuv2yuv1 = RENAME(yuv2yuv1    );
2202                 c->yuv2yuvX = RENAME(yuv2yuvX    );
2203                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
2204                     switch (c->dstFormat) {
2205                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;
2206                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;
2207                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;
2208                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;
2209                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;
2210                     default: break;
2211                     }
2212                 }
2213             }
2214         }
2215         if (!(c->flags & SWS_FULL_CHR_H_INT)) {
2216             switch (c->dstFormat) {
2217             case PIX_FMT_RGB32:
2218                 c->yuv2packed1 = RENAME(yuv2rgb32_1);
2219                 c->yuv2packed2 = RENAME(yuv2rgb32_2);
2220                 break;
2221             case PIX_FMT_BGR24:
2222                 c->yuv2packed1 = RENAME(yuv2bgr24_1);
2223                 c->yuv2packed2 = RENAME(yuv2bgr24_2);
2224                 break;
2225             case PIX_FMT_RGB555:
2226                 c->yuv2packed1 = RENAME(yuv2rgb555_1);
2227                 c->yuv2packed2 = RENAME(yuv2rgb555_2);
2228                 break;
2229             case PIX_FMT_RGB565:
2230                 c->yuv2packed1 = RENAME(yuv2rgb565_1);
2231                 c->yuv2packed2 = RENAME(yuv2rgb565_2);
2232                 break;
2233             case PIX_FMT_YUYV422:
2234                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);
2235                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);
2236                 break;
2237             default:
2238                 break;
2239             }
2240         }
2241     }
2242
2243 #if !COMPILE_TEMPLATE_MMX2
2244     c->hScale       = RENAME(hScale      );
2245 #endif /* !COMPILE_TEMPLATE_MMX2 */
2246
2247     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2248 #if COMPILE_TEMPLATE_MMX2
2249     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
2250     {
2251         c->hyscale_fast = RENAME(hyscale_fast);
2252         c->hcscale_fast = RENAME(hcscale_fast);
2253     } else {
2254 #endif /* COMPILE_TEMPLATE_MMX2 */
2255         c->hyscale_fast = NULL;
2256         c->hcscale_fast = NULL;
2257 #if COMPILE_TEMPLATE_MMX2
2258     }
2259 #endif /* COMPILE_TEMPLATE_MMX2 */
2260
2261 #if !COMPILE_TEMPLATE_MMX2
2262     switch(srcFormat) {
2263         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;
2264         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;
2265         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;
2266         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;
2267         case PIX_FMT_YUV420P16BE:
2268         case PIX_FMT_YUV422P16BE:
2269         case PIX_FMT_YUV444P16BE: c->chrToYV12 = RENAME(BEToUV); break;
2270         case PIX_FMT_YUV420P16LE:
2271         case PIX_FMT_YUV422P16LE:
2272         case PIX_FMT_YUV444P16LE: c->chrToYV12 = RENAME(LEToUV); break;
2273         default: break;
2274     }
2275 #endif /* !COMPILE_TEMPLATE_MMX2 */
2276     if (!c->chrSrcHSubSample) {
2277         switch(srcFormat) {
2278         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
2279         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
2280         default: break;
2281         }
2282     }
2283
2284     switch (srcFormat) {
2285 #if !COMPILE_TEMPLATE_MMX2
2286     case PIX_FMT_YUYV422  :
2287     case PIX_FMT_YUV420P16BE:
2288     case PIX_FMT_YUV422P16BE:
2289     case PIX_FMT_YUV444P16BE:
2290     case PIX_FMT_Y400A    :
2291     case PIX_FMT_GRAY16BE : c->lumToYV12 = RENAME(yuy2ToY); break;
2292     case PIX_FMT_UYVY422  :
2293     case PIX_FMT_YUV420P16LE:
2294     case PIX_FMT_YUV422P16LE:
2295     case PIX_FMT_YUV444P16LE:
2296     case PIX_FMT_GRAY16LE : c->lumToYV12 = RENAME(uyvyToY); break;
2297 #endif /* !COMPILE_TEMPLATE_MMX2 */
2298     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
2299     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
2300     default: break;
2301     }
2302 #if !COMPILE_TEMPLATE_MMX2
2303     if (c->alpPixBuf) {
2304         switch (srcFormat) {
2305         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;
2306         default: break;
2307         }
2308     }
2309 #endif /* !COMPILE_TEMPLATE_MMX2 */
2310 }