swscale: reformat x86/swscale_template.c.
[ffmpeg.git] / libswscale / x86 / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of Libav.
5  *
6  * Libav is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * Libav is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with Libav; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #include "swscale_template.h"
22
23 #undef REAL_MOVNTQ
24 #undef MOVNTQ
25 #undef PREFETCH
26
27 #if COMPILE_TEMPLATE_MMX2
28 #define PREFETCH "prefetchnta"
29 #else
30 #define PREFETCH  " # nop"
31 #endif
32
33 #if COMPILE_TEMPLATE_MMX2
34 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
35 #else
36 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
37 #endif
38 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
39
40 #define YSCALEYUV2YV12X(x, offset, dest, width) \
41     __asm__ volatile(\
42         "xor                          %%"REG_a", %%"REG_a"  \n\t"\
43         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
44         "movq                             %%mm3, %%mm4      \n\t"\
45         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
46         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
47         ".p2align                             4             \n\t" /* FIXME Unroll? */\
48         "1:                                                 \n\t"\
49         "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\
50         "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm2      \n\t" /* srcData */\
51         "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm5      \n\t" /* srcData */\
52         "add                                $16, %%"REG_d"  \n\t"\
53         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
54         "test                         %%"REG_S", %%"REG_S"  \n\t"\
55         "pmulhw                           %%mm0, %%mm2      \n\t"\
56         "pmulhw                           %%mm0, %%mm5      \n\t"\
57         "paddw                            %%mm2, %%mm3      \n\t"\
58         "paddw                            %%mm5, %%mm4      \n\t"\
59         " jnz                                1b             \n\t"\
60         "psraw                               $3, %%mm3      \n\t"\
61         "psraw                               $3, %%mm4      \n\t"\
62         "packuswb                         %%mm4, %%mm3      \n\t"\
63         MOVNTQ(%%mm3, (%1, %%REGa))\
64         "add                                 $8, %%"REG_a"  \n\t"\
65         "cmp                                 %2, %%"REG_a"  \n\t"\
66         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
67         "movq                             %%mm3, %%mm4      \n\t"\
68         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
69         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
70         "jb                                  1b             \n\t"\
71         :: "r" (&c->redDither),\
72         "r" (dest), "g" ((x86_reg)width)\
73         : "%"REG_a, "%"REG_d, "%"REG_S\
74     );
75
76 static inline void RENAME(yuv2yuvX)(SwsContext *c, const int16_t *lumFilter,
77                                     const int16_t **lumSrc, int lumFilterSize,
78                                     const int16_t *chrFilter, const int16_t **chrSrc,
79                                     int chrFilterSize, const int16_t **alpSrc,
80                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
81                                     uint8_t *aDest, long dstW, long chrDstW)
82 {
83     if (uDest) {
84         YSCALEYUV2YV12X(   "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
85         YSCALEYUV2YV12X(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
86     }
87     if (CONFIG_SWSCALE_ALPHA && aDest) {
88         YSCALEYUV2YV12X(   "0", ALP_MMX_FILTER_OFFSET, aDest, dstW)
89     }
90
91     YSCALEYUV2YV12X("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
92 }
93
94 #define YSCALEYUV2YV12X_ACCURATE(x, offset, dest, width) \
95     __asm__ volatile(\
96         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
97         "xor                          %%"REG_a", %%"REG_a"  \n\t"\
98         "pxor                             %%mm4, %%mm4      \n\t"\
99         "pxor                             %%mm5, %%mm5      \n\t"\
100         "pxor                             %%mm6, %%mm6      \n\t"\
101         "pxor                             %%mm7, %%mm7      \n\t"\
102         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
103         ".p2align                             4             \n\t"\
104         "1:                                                 \n\t"\
105         "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm0      \n\t" /* srcData */\
106         "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm2      \n\t" /* srcData */\
107         "mov        "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"  \n\t"\
108         "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm1      \n\t" /* srcData */\
109         "movq                             %%mm0, %%mm3      \n\t"\
110         "punpcklwd                        %%mm1, %%mm0      \n\t"\
111         "punpckhwd                        %%mm1, %%mm3      \n\t"\
112         "movq       "STR(APCK_COEF)"(%%"REG_d"), %%mm1      \n\t" /* filterCoeff */\
113         "pmaddwd                          %%mm1, %%mm0      \n\t"\
114         "pmaddwd                          %%mm1, %%mm3      \n\t"\
115         "paddd                            %%mm0, %%mm4      \n\t"\
116         "paddd                            %%mm3, %%mm5      \n\t"\
117         "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm3      \n\t" /* srcData */\
118         "mov        "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"  \n\t"\
119         "add                  $"STR(APCK_SIZE)", %%"REG_d"  \n\t"\
120         "test                         %%"REG_S", %%"REG_S"  \n\t"\
121         "movq                             %%mm2, %%mm0      \n\t"\
122         "punpcklwd                        %%mm3, %%mm2      \n\t"\
123         "punpckhwd                        %%mm3, %%mm0      \n\t"\
124         "pmaddwd                          %%mm1, %%mm2      \n\t"\
125         "pmaddwd                          %%mm1, %%mm0      \n\t"\
126         "paddd                            %%mm2, %%mm6      \n\t"\
127         "paddd                            %%mm0, %%mm7      \n\t"\
128         " jnz                                1b             \n\t"\
129         "psrad                              $16, %%mm4      \n\t"\
130         "psrad                              $16, %%mm5      \n\t"\
131         "psrad                              $16, %%mm6      \n\t"\
132         "psrad                              $16, %%mm7      \n\t"\
133         "movq             "VROUNDER_OFFSET"(%0), %%mm0      \n\t"\
134         "packssdw                         %%mm5, %%mm4      \n\t"\
135         "packssdw                         %%mm7, %%mm6      \n\t"\
136         "paddw                            %%mm0, %%mm4      \n\t"\
137         "paddw                            %%mm0, %%mm6      \n\t"\
138         "psraw                               $3, %%mm4      \n\t"\
139         "psraw                               $3, %%mm6      \n\t"\
140         "packuswb                         %%mm6, %%mm4      \n\t"\
141         MOVNTQ(%%mm4, (%1, %%REGa))\
142         "add                                 $8, %%"REG_a"  \n\t"\
143         "cmp                                 %2, %%"REG_a"  \n\t"\
144         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
145         "pxor                             %%mm4, %%mm4      \n\t"\
146         "pxor                             %%mm5, %%mm5      \n\t"\
147         "pxor                             %%mm6, %%mm6      \n\t"\
148         "pxor                             %%mm7, %%mm7      \n\t"\
149         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
150         "jb                                  1b             \n\t"\
151         :: "r" (&c->redDither),\
152         "r" (dest), "g" ((x86_reg)width)\
153         : "%"REG_a, "%"REG_d, "%"REG_S\
154     );
155
156 static inline void RENAME(yuv2yuvX_ar)(SwsContext *c, const int16_t *lumFilter,
157                                        const int16_t **lumSrc, int lumFilterSize,
158                                        const int16_t *chrFilter, const int16_t **chrSrc,
159                                        int chrFilterSize, const int16_t **alpSrc,
160                                        uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
161                                        uint8_t *aDest, long dstW, long chrDstW)
162 {
163     if (uDest) {
164         YSCALEYUV2YV12X_ACCURATE(   "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
165         YSCALEYUV2YV12X_ACCURATE(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
166     }
167     if (CONFIG_SWSCALE_ALPHA && aDest) {
168         YSCALEYUV2YV12X_ACCURATE(   "0", ALP_MMX_FILTER_OFFSET, aDest, dstW)
169     }
170
171     YSCALEYUV2YV12X_ACCURATE("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
172 }
173
174 #define YSCALEYUV2YV121 \
175     "mov %2, %%"REG_a"                    \n\t"\
176     ".p2align               4             \n\t" /* FIXME Unroll? */\
177     "1:                                   \n\t"\
178     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
179     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
180     "psraw                 $7, %%mm0      \n\t"\
181     "psraw                 $7, %%mm1      \n\t"\
182     "packuswb           %%mm1, %%mm0      \n\t"\
183     MOVNTQ(%%mm0, (%1, %%REGa))\
184     "add                   $8, %%"REG_a"  \n\t"\
185     "jnc                   1b             \n\t"
186
187 static inline void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc,
188                                     const int16_t *chrSrc, const int16_t *alpSrc,
189                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
190                                     uint8_t *aDest, long dstW, long chrDstW)
191 {
192     long p= 4;
193     const uint8_t *src[4]= { alpSrc + dstW, lumSrc + dstW, chrSrc + chrDstW, chrSrc + VOFW + chrDstW };
194     uint8_t *dst[4]= { aDest, dest, uDest, vDest };
195     x86_reg counter[4]= { dstW, dstW, chrDstW, chrDstW };
196
197     while (p--) {
198         if (dst[p]) {
199             __asm__ volatile(
200                YSCALEYUV2YV121
201                :: "r" (src[p]), "r" (dst[p] + counter[p]),
202                   "g" (-counter[p])
203                : "%"REG_a
204             );
205         }
206     }
207 }
208
209 #define YSCALEYUV2YV121_ACCURATE \
210     "mov %2, %%"REG_a"                    \n\t"\
211     "pcmpeqw %%mm7, %%mm7                 \n\t"\
212     "psrlw                 $15, %%mm7     \n\t"\
213     "psllw                  $6, %%mm7     \n\t"\
214     ".p2align                4            \n\t" /* FIXME Unroll? */\
215     "1:                                   \n\t"\
216     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
217     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
218     "paddsw             %%mm7, %%mm0      \n\t"\
219     "paddsw             %%mm7, %%mm1      \n\t"\
220     "psraw                 $7, %%mm0      \n\t"\
221     "psraw                 $7, %%mm1      \n\t"\
222     "packuswb           %%mm1, %%mm0      \n\t"\
223     MOVNTQ(%%mm0, (%1, %%REGa))\
224     "add                   $8, %%"REG_a"  \n\t"\
225     "jnc                   1b             \n\t"
226
227 static inline void RENAME(yuv2yuv1_ar)(SwsContext *c, const int16_t *lumSrc,
228                                        const int16_t *chrSrc, const int16_t *alpSrc,
229                                        uint8_t *dest, uint8_t *uDest, uint8_t *vDest,
230                                        uint8_t *aDest, long dstW, long chrDstW)
231 {
232     long p= 4;
233     const uint8_t *src[4]= { alpSrc + dstW, lumSrc + dstW, chrSrc + chrDstW, chrSrc + VOFW + chrDstW };
234     uint8_t *dst[4]= { aDest, dest, uDest, vDest };
235     x86_reg counter[4]= { dstW, dstW, chrDstW, chrDstW };
236
237     while (p--) {
238         if (dst[p]) {
239             __asm__ volatile(
240                 YSCALEYUV2YV121_ACCURATE
241                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
242                    "g" (-counter[p])
243                 : "%"REG_a
244             );
245         }
246     }
247 }
248
249 #define YSCALEYUV2PACKEDX_UV \
250     __asm__ volatile(\
251         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
252         ".p2align                      4                \n\t"\
253         "nop                                            \n\t"\
254         "1:                                             \n\t"\
255         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
256         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
257         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
258         "movq                      %%mm3, %%mm4         \n\t"\
259         ".p2align                      4                \n\t"\
260         "2:                                             \n\t"\
261         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
262         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
263         "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
264         "add                         $16, %%"REG_d"     \n\t"\
265         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
266         "pmulhw                    %%mm0, %%mm2         \n\t"\
267         "pmulhw                    %%mm0, %%mm5         \n\t"\
268         "paddw                     %%mm2, %%mm3         \n\t"\
269         "paddw                     %%mm5, %%mm4         \n\t"\
270         "test                  %%"REG_S", %%"REG_S"     \n\t"\
271         " jnz                         2b                \n\t"\
272
273 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
274     "lea                "offset"(%0), %%"REG_d"     \n\t"\
275     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
276     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
277     "movq                    "#dst1", "#dst2"       \n\t"\
278     ".p2align                      4                \n\t"\
279     "2:                                             \n\t"\
280     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
281     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
282     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
283     "add                         $16, %%"REG_d"            \n\t"\
284     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
285     "pmulhw                 "#coeff", "#src1"       \n\t"\
286     "pmulhw                 "#coeff", "#src2"       \n\t"\
287     "paddw                   "#src1", "#dst1"       \n\t"\
288     "paddw                   "#src2", "#dst2"       \n\t"\
289     "test                  %%"REG_S", %%"REG_S"     \n\t"\
290     " jnz                         2b                \n\t"\
291
292 #define YSCALEYUV2PACKEDX \
293     YSCALEYUV2PACKEDX_UV \
294     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
295
296 #define YSCALEYUV2PACKEDX_END                     \
297         :: "r" (&c->redDither),                   \
298             "m" (dummy), "m" (dummy), "m" (dummy),\
299             "r" (dest), "m" (dstW_reg)            \
300         : "%"REG_a, "%"REG_d, "%"REG_S            \
301     );
302
303 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
304     __asm__ volatile(\
305         "xor %%"REG_a", %%"REG_a"                       \n\t"\
306         ".p2align                      4                \n\t"\
307         "nop                                            \n\t"\
308         "1:                                             \n\t"\
309         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
310         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
311         "pxor                      %%mm4, %%mm4         \n\t"\
312         "pxor                      %%mm5, %%mm5         \n\t"\
313         "pxor                      %%mm6, %%mm6         \n\t"\
314         "pxor                      %%mm7, %%mm7         \n\t"\
315         ".p2align                      4                \n\t"\
316         "2:                                             \n\t"\
317         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
318         "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
319         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
320         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
321         "movq                      %%mm0, %%mm3         \n\t"\
322         "punpcklwd                 %%mm1, %%mm0         \n\t"\
323         "punpckhwd                 %%mm1, %%mm3         \n\t"\
324         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
325         "pmaddwd                   %%mm1, %%mm0         \n\t"\
326         "pmaddwd                   %%mm1, %%mm3         \n\t"\
327         "paddd                     %%mm0, %%mm4         \n\t"\
328         "paddd                     %%mm3, %%mm5         \n\t"\
329         "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
330         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
331         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
332         "test                  %%"REG_S", %%"REG_S"     \n\t"\
333         "movq                      %%mm2, %%mm0         \n\t"\
334         "punpcklwd                 %%mm3, %%mm2         \n\t"\
335         "punpckhwd                 %%mm3, %%mm0         \n\t"\
336         "pmaddwd                   %%mm1, %%mm2         \n\t"\
337         "pmaddwd                   %%mm1, %%mm0         \n\t"\
338         "paddd                     %%mm2, %%mm6         \n\t"\
339         "paddd                     %%mm0, %%mm7         \n\t"\
340         " jnz                         2b                \n\t"\
341         "psrad                       $16, %%mm4         \n\t"\
342         "psrad                       $16, %%mm5         \n\t"\
343         "psrad                       $16, %%mm6         \n\t"\
344         "psrad                       $16, %%mm7         \n\t"\
345         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
346         "packssdw                  %%mm5, %%mm4         \n\t"\
347         "packssdw                  %%mm7, %%mm6         \n\t"\
348         "paddw                     %%mm0, %%mm4         \n\t"\
349         "paddw                     %%mm0, %%mm6         \n\t"\
350         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
351         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
352
353 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
354     "lea                "offset"(%0), %%"REG_d"     \n\t"\
355     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
356     "pxor                      %%mm1, %%mm1         \n\t"\
357     "pxor                      %%mm5, %%mm5         \n\t"\
358     "pxor                      %%mm7, %%mm7         \n\t"\
359     "pxor                      %%mm6, %%mm6         \n\t"\
360     ".p2align                      4                \n\t"\
361     "2:                                             \n\t"\
362     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
363     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
364     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
365     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
366     "movq                      %%mm0, %%mm3         \n\t"\
367     "punpcklwd                 %%mm4, %%mm0         \n\t"\
368     "punpckhwd                 %%mm4, %%mm3         \n\t"\
369     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
370     "pmaddwd                   %%mm4, %%mm0         \n\t"\
371     "pmaddwd                   %%mm4, %%mm3         \n\t"\
372     "paddd                     %%mm0, %%mm1         \n\t"\
373     "paddd                     %%mm3, %%mm5         \n\t"\
374     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
375     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
376     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
377     "test                  %%"REG_S", %%"REG_S"     \n\t"\
378     "movq                      %%mm2, %%mm0         \n\t"\
379     "punpcklwd                 %%mm3, %%mm2         \n\t"\
380     "punpckhwd                 %%mm3, %%mm0         \n\t"\
381     "pmaddwd                   %%mm4, %%mm2         \n\t"\
382     "pmaddwd                   %%mm4, %%mm0         \n\t"\
383     "paddd                     %%mm2, %%mm7         \n\t"\
384     "paddd                     %%mm0, %%mm6         \n\t"\
385     " jnz                         2b                \n\t"\
386     "psrad                       $16, %%mm1         \n\t"\
387     "psrad                       $16, %%mm5         \n\t"\
388     "psrad                       $16, %%mm7         \n\t"\
389     "psrad                       $16, %%mm6         \n\t"\
390     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
391     "packssdw                  %%mm5, %%mm1         \n\t"\
392     "packssdw                  %%mm6, %%mm7         \n\t"\
393     "paddw                     %%mm0, %%mm1         \n\t"\
394     "paddw                     %%mm0, %%mm7         \n\t"\
395     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
396     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
397
398 #define YSCALEYUV2PACKEDX_ACCURATE \
399     YSCALEYUV2PACKEDX_ACCURATE_UV \
400     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
401
402 #define YSCALEYUV2RGBX \
403     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
404     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
405     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
406     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
407     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
408     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
409     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
410     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
411     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
412     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
413     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
414     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
415     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
416     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
417     "paddw           %%mm3, %%mm4       \n\t"\
418     "movq            %%mm2, %%mm0       \n\t"\
419     "movq            %%mm5, %%mm6       \n\t"\
420     "movq            %%mm4, %%mm3       \n\t"\
421     "punpcklwd       %%mm2, %%mm2       \n\t"\
422     "punpcklwd       %%mm5, %%mm5       \n\t"\
423     "punpcklwd       %%mm4, %%mm4       \n\t"\
424     "paddw           %%mm1, %%mm2       \n\t"\
425     "paddw           %%mm1, %%mm5       \n\t"\
426     "paddw           %%mm1, %%mm4       \n\t"\
427     "punpckhwd       %%mm0, %%mm0       \n\t"\
428     "punpckhwd       %%mm6, %%mm6       \n\t"\
429     "punpckhwd       %%mm3, %%mm3       \n\t"\
430     "paddw           %%mm7, %%mm0       \n\t"\
431     "paddw           %%mm7, %%mm6       \n\t"\
432     "paddw           %%mm7, %%mm3       \n\t"\
433     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
434     "packuswb        %%mm0, %%mm2       \n\t"\
435     "packuswb        %%mm6, %%mm5       \n\t"\
436     "packuswb        %%mm3, %%mm4       \n\t"\
437
438 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
439     "movq       "#b", "#q2"     \n\t" /* B */\
440     "movq       "#r", "#t"      \n\t" /* R */\
441     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
442     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
443     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
444     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
445     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
446     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
447     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
448     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
449     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
450     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
451 \
452     MOVNTQ(   q0,   (dst, index, 4))\
453     MOVNTQ(    b,  8(dst, index, 4))\
454     MOVNTQ(   q2, 16(dst, index, 4))\
455     MOVNTQ(   q3, 24(dst, index, 4))\
456 \
457     "add      $8, "#index"      \n\t"\
458     "cmp "#dstw", "#index"      \n\t"\
459     " jb      1b                \n\t"
460 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
461
462 static inline void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,
463                                           const int16_t **lumSrc, int lumFilterSize,
464                                           const int16_t *chrFilter, const int16_t **chrSrc,
465                                           int chrFilterSize, const int16_t **alpSrc,
466                                           uint8_t *dest, long dstW, long dstY)
467 {
468     x86_reg dummy=0;
469     x86_reg dstW_reg = dstW;
470
471     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
472         YSCALEYUV2PACKEDX_ACCURATE
473         YSCALEYUV2RGBX
474         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
475         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
476         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
477         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
478         "movq               "Y_TEMP"(%0), %%mm5         \n\t"
479         "psraw                        $3, %%mm1         \n\t"
480         "psraw                        $3, %%mm7         \n\t"
481         "packuswb                  %%mm7, %%mm1         \n\t"
482         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
483         YSCALEYUV2PACKEDX_END
484     } else {
485         YSCALEYUV2PACKEDX_ACCURATE
486         YSCALEYUV2RGBX
487         "pcmpeqd %%mm7, %%mm7 \n\t"
488         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
489         YSCALEYUV2PACKEDX_END
490     }
491 }
492
493 static inline void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,
494                                        const int16_t **lumSrc, int lumFilterSize,
495                                        const int16_t *chrFilter, const int16_t **chrSrc,
496                                        int chrFilterSize, const int16_t **alpSrc,
497                                        uint8_t *dest, long dstW, long dstY)
498 {
499     x86_reg dummy=0;
500     x86_reg dstW_reg = dstW;
501
502     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
503         YSCALEYUV2PACKEDX
504         YSCALEYUV2RGBX
505         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
506         "psraw                        $3, %%mm1         \n\t"
507         "psraw                        $3, %%mm7         \n\t"
508         "packuswb                  %%mm7, %%mm1         \n\t"
509         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
510         YSCALEYUV2PACKEDX_END
511     } else {
512         YSCALEYUV2PACKEDX
513         YSCALEYUV2RGBX
514         "pcmpeqd %%mm7, %%mm7 \n\t"
515         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
516         YSCALEYUV2PACKEDX_END
517     }
518 }
519
520 #define REAL_WRITERGB16(dst, dstw, index) \
521     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
522     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
523     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
524     "psrlq           $3, %%mm2  \n\t"\
525 \
526     "movq         %%mm2, %%mm1  \n\t"\
527     "movq         %%mm4, %%mm3  \n\t"\
528 \
529     "punpcklbw    %%mm7, %%mm3  \n\t"\
530     "punpcklbw    %%mm5, %%mm2  \n\t"\
531     "punpckhbw    %%mm7, %%mm4  \n\t"\
532     "punpckhbw    %%mm5, %%mm1  \n\t"\
533 \
534     "psllq           $3, %%mm3  \n\t"\
535     "psllq           $3, %%mm4  \n\t"\
536 \
537     "por          %%mm3, %%mm2  \n\t"\
538     "por          %%mm4, %%mm1  \n\t"\
539 \
540     MOVNTQ(%%mm2,  (dst, index, 2))\
541     MOVNTQ(%%mm1, 8(dst, index, 2))\
542 \
543     "add             $8, "#index"   \n\t"\
544     "cmp        "#dstw", "#index"   \n\t"\
545     " jb             1b             \n\t"
546 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
547
548 static inline void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,
549                                            const int16_t **lumSrc, int lumFilterSize,
550                                            const int16_t *chrFilter, const int16_t **chrSrc,
551                                            int chrFilterSize, const int16_t **alpSrc,
552                                            uint8_t *dest, long dstW, long dstY)
553 {
554     x86_reg dummy=0;
555     x86_reg dstW_reg = dstW;
556
557     YSCALEYUV2PACKEDX_ACCURATE
558     YSCALEYUV2RGBX
559     "pxor %%mm7, %%mm7 \n\t"
560     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
561 #ifdef DITHER1XBPP
562     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
563     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
564     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
565 #endif
566     WRITERGB16(%4, %5, %%REGa)
567     YSCALEYUV2PACKEDX_END
568 }
569
570 static inline void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,
571                                         const int16_t **lumSrc, int lumFilterSize,
572                                         const int16_t *chrFilter, const int16_t **chrSrc,
573                                         int chrFilterSize, const int16_t **alpSrc,
574                                         uint8_t *dest, long dstW, long dstY)
575 {
576     x86_reg dummy=0;
577     x86_reg dstW_reg = dstW;
578
579     YSCALEYUV2PACKEDX
580     YSCALEYUV2RGBX
581     "pxor %%mm7, %%mm7 \n\t"
582     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
583 #ifdef DITHER1XBPP
584     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
585     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
586     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
587 #endif
588     WRITERGB16(%4, %5, %%REGa)
589     YSCALEYUV2PACKEDX_END
590 }
591
592 #define REAL_WRITERGB15(dst, dstw, index) \
593     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
594     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
595     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
596     "psrlq           $3, %%mm2  \n\t"\
597     "psrlq           $1, %%mm5  \n\t"\
598 \
599     "movq         %%mm2, %%mm1  \n\t"\
600     "movq         %%mm4, %%mm3  \n\t"\
601 \
602     "punpcklbw    %%mm7, %%mm3  \n\t"\
603     "punpcklbw    %%mm5, %%mm2  \n\t"\
604     "punpckhbw    %%mm7, %%mm4  \n\t"\
605     "punpckhbw    %%mm5, %%mm1  \n\t"\
606 \
607     "psllq           $2, %%mm3  \n\t"\
608     "psllq           $2, %%mm4  \n\t"\
609 \
610     "por          %%mm3, %%mm2  \n\t"\
611     "por          %%mm4, %%mm1  \n\t"\
612 \
613     MOVNTQ(%%mm2,  (dst, index, 2))\
614     MOVNTQ(%%mm1, 8(dst, index, 2))\
615 \
616     "add             $8, "#index"   \n\t"\
617     "cmp        "#dstw", "#index"   \n\t"\
618     " jb             1b             \n\t"
619 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
620
621 static inline void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,
622                                            const int16_t **lumSrc, int lumFilterSize,
623                                            const int16_t *chrFilter, const int16_t **chrSrc,
624                                            int chrFilterSize, const int16_t **alpSrc,
625                                            uint8_t *dest, long dstW, long dstY)
626 {
627     x86_reg dummy=0;
628     x86_reg dstW_reg = dstW;
629
630     YSCALEYUV2PACKEDX_ACCURATE
631     YSCALEYUV2RGBX
632     "pxor %%mm7, %%mm7 \n\t"
633     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
634 #ifdef DITHER1XBPP
635     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
636     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
637     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
638 #endif
639     WRITERGB15(%4, %5, %%REGa)
640     YSCALEYUV2PACKEDX_END
641 }
642
643 static inline void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,
644                                         const int16_t **lumSrc, int lumFilterSize,
645                                         const int16_t *chrFilter, const int16_t **chrSrc,
646                                         int chrFilterSize, const int16_t **alpSrc,
647                                         uint8_t *dest, long dstW, long dstY)
648 {
649     x86_reg dummy=0;
650     x86_reg dstW_reg = dstW;
651
652     YSCALEYUV2PACKEDX
653     YSCALEYUV2RGBX
654     "pxor %%mm7, %%mm7 \n\t"
655     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
656 #ifdef DITHER1XBPP
657     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
658     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
659     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
660 #endif
661     WRITERGB15(%4, %5, %%REGa)
662     YSCALEYUV2PACKEDX_END
663 }
664
665 #define WRITEBGR24MMX(dst, dstw, index) \
666     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
667     "movq      %%mm2, %%mm1     \n\t" /* B */\
668     "movq      %%mm5, %%mm6     \n\t" /* R */\
669     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
670     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
671     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
672     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
673     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
674     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
675     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
676     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
677     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
678     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
679 \
680     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
681     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
682     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
683     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
684 \
685     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
686     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
687     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
688     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
689 \
690     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
691     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
692     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
693     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
694 \
695     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
696     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
697     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
698     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
699     MOVNTQ(%%mm0, (dst))\
700 \
701     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
702     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
703     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
704     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
705     MOVNTQ(%%mm6, 8(dst))\
706 \
707     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
708     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
709     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
710     MOVNTQ(%%mm5, 16(dst))\
711 \
712     "add         $24, "#dst"    \n\t"\
713 \
714     "add          $8, "#index"  \n\t"\
715     "cmp     "#dstw", "#index"  \n\t"\
716     " jb          1b            \n\t"
717
718 #define WRITEBGR24MMX2(dst, dstw, index) \
719     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
720     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
721     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
722     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
723     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
724     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
725 \
726     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
727     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
728     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
729 \
730     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
731     "por    %%mm1, %%mm6        \n\t"\
732     "por    %%mm3, %%mm6        \n\t"\
733     MOVNTQ(%%mm6, (dst))\
734 \
735     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
736     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
737     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
738     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
739 \
740     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
741     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
742     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
743 \
744     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
745     "por    %%mm3, %%mm6        \n\t"\
746     MOVNTQ(%%mm6, 8(dst))\
747 \
748     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
749     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
750     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
751 \
752     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
753     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
754     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
755 \
756     "por    %%mm1, %%mm3        \n\t"\
757     "por    %%mm3, %%mm6        \n\t"\
758     MOVNTQ(%%mm6, 16(dst))\
759 \
760     "add      $24, "#dst"       \n\t"\
761 \
762     "add       $8, "#index"     \n\t"\
763     "cmp  "#dstw", "#index"     \n\t"\
764     " jb       1b               \n\t"
765
766 #if COMPILE_TEMPLATE_MMX2
767 #undef WRITEBGR24
768 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
769 #else
770 #undef WRITEBGR24
771 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
772 #endif
773
774 static inline void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,
775                                           const int16_t **lumSrc, int lumFilterSize,
776                                           const int16_t *chrFilter, const int16_t **chrSrc,
777                                           int chrFilterSize, const int16_t **alpSrc,
778                                           uint8_t *dest, long dstW, long dstY)
779 {
780     x86_reg dummy=0;
781     x86_reg dstW_reg = dstW;
782
783     YSCALEYUV2PACKEDX_ACCURATE
784     YSCALEYUV2RGBX
785     "pxor %%mm7, %%mm7 \n\t"
786     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
787     "add %4, %%"REG_c"                        \n\t"
788     WRITEBGR24(%%REGc, %5, %%REGa)
789     :: "r" (&c->redDither),
790        "m" (dummy), "m" (dummy), "m" (dummy),
791        "r" (dest), "m" (dstW_reg)
792     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
793     );
794 }
795
796 static inline void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,
797                                        const int16_t **lumSrc, int lumFilterSize,
798                                        const int16_t *chrFilter, const int16_t **chrSrc,
799                                        int chrFilterSize, const int16_t **alpSrc,
800                                        uint8_t *dest, long dstW, long dstY)
801 {
802     x86_reg dummy=0;
803     x86_reg dstW_reg = dstW;
804
805     YSCALEYUV2PACKEDX
806     YSCALEYUV2RGBX
807     "pxor                    %%mm7, %%mm7       \n\t"
808     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
809     "add                        %4, %%"REG_c"   \n\t"
810     WRITEBGR24(%%REGc, %5, %%REGa)
811     :: "r" (&c->redDither),
812        "m" (dummy), "m" (dummy), "m" (dummy),
813        "r" (dest),  "m" (dstW_reg)
814     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
815     );
816 }
817
818 #define REAL_WRITEYUY2(dst, dstw, index) \
819     "packuswb  %%mm3, %%mm3     \n\t"\
820     "packuswb  %%mm4, %%mm4     \n\t"\
821     "packuswb  %%mm7, %%mm1     \n\t"\
822     "punpcklbw %%mm4, %%mm3     \n\t"\
823     "movq      %%mm1, %%mm7     \n\t"\
824     "punpcklbw %%mm3, %%mm1     \n\t"\
825     "punpckhbw %%mm3, %%mm7     \n\t"\
826 \
827     MOVNTQ(%%mm1, (dst, index, 2))\
828     MOVNTQ(%%mm7, 8(dst, index, 2))\
829 \
830     "add          $8, "#index"  \n\t"\
831     "cmp     "#dstw", "#index"  \n\t"\
832     " jb          1b            \n\t"
833 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
834
835 static inline void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,
836                                             const int16_t **lumSrc, int lumFilterSize,
837                                             const int16_t *chrFilter, const int16_t **chrSrc,
838                                             int chrFilterSize, const int16_t **alpSrc,
839                                             uint8_t *dest, long dstW, long dstY)
840 {
841     x86_reg dummy=0;
842     x86_reg dstW_reg = dstW;
843
844     YSCALEYUV2PACKEDX_ACCURATE
845     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
846     "psraw $3, %%mm3    \n\t"
847     "psraw $3, %%mm4    \n\t"
848     "psraw $3, %%mm1    \n\t"
849     "psraw $3, %%mm7    \n\t"
850     WRITEYUY2(%4, %5, %%REGa)
851     YSCALEYUV2PACKEDX_END
852 }
853
854 static inline void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,
855                                          const int16_t **lumSrc, int lumFilterSize,
856                                          const int16_t *chrFilter, const int16_t **chrSrc,
857                                          int chrFilterSize, const int16_t **alpSrc,
858                                          uint8_t *dest, long dstW, long dstY)
859 {
860     x86_reg dummy=0;
861     x86_reg dstW_reg = dstW;
862
863     YSCALEYUV2PACKEDX
864     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
865     "psraw $3, %%mm3    \n\t"
866     "psraw $3, %%mm4    \n\t"
867     "psraw $3, %%mm1    \n\t"
868     "psraw $3, %%mm7    \n\t"
869     WRITEYUY2(%4, %5, %%REGa)
870     YSCALEYUV2PACKEDX_END
871 }
872
873 #define REAL_YSCALEYUV2RGB_UV(index, c) \
874     "xor            "#index", "#index"  \n\t"\
875     ".p2align              4            \n\t"\
876     "1:                                 \n\t"\
877     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
878     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
879     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
880     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
881     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
882     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
883     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
884     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
885     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
886     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
887     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
888     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
889     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
890     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
891     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
892     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
893     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
894     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
895     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
896     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
897
898 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
899     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
900     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
901     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
902     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
903     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
904     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
905     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
906     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
907     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
908     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
909     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
910     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
911
912 #define REAL_YSCALEYUV2RGB_COEFF(c) \
913     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
914     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
915     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
916     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
917     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
918     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
919     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
920     "paddw             %%mm3, %%mm4     \n\t"\
921     "movq              %%mm2, %%mm0     \n\t"\
922     "movq              %%mm5, %%mm6     \n\t"\
923     "movq              %%mm4, %%mm3     \n\t"\
924     "punpcklwd         %%mm2, %%mm2     \n\t"\
925     "punpcklwd         %%mm5, %%mm5     \n\t"\
926     "punpcklwd         %%mm4, %%mm4     \n\t"\
927     "paddw             %%mm1, %%mm2     \n\t"\
928     "paddw             %%mm1, %%mm5     \n\t"\
929     "paddw             %%mm1, %%mm4     \n\t"\
930     "punpckhwd         %%mm0, %%mm0     \n\t"\
931     "punpckhwd         %%mm6, %%mm6     \n\t"\
932     "punpckhwd         %%mm3, %%mm3     \n\t"\
933     "paddw             %%mm7, %%mm0     \n\t"\
934     "paddw             %%mm7, %%mm6     \n\t"\
935     "paddw             %%mm7, %%mm3     \n\t"\
936     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
937     "packuswb          %%mm0, %%mm2     \n\t"\
938     "packuswb          %%mm6, %%mm5     \n\t"\
939     "packuswb          %%mm3, %%mm4     \n\t"\
940
941 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
942
943 #define YSCALEYUV2RGB(index, c) \
944     REAL_YSCALEYUV2RGB_UV(index, c) \
945     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
946     REAL_YSCALEYUV2RGB_COEFF(c)
947
948 /**
949  * vertical bilinear scale YV12 to RGB
950  */
951 static inline void RENAME(yuv2rgb32_2)(SwsContext *c, const uint16_t *buf0,
952                                        const uint16_t *buf1, const uint16_t *uvbuf0,
953                                        const uint16_t *uvbuf1, const uint16_t *abuf0,
954                                        const uint16_t *abuf1, uint8_t *dest,
955                                        int dstW, int yalpha, int uvalpha, int y)
956 {
957     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
958 #if ARCH_X86_64
959         __asm__ volatile(
960             YSCALEYUV2RGB(%%r8, %5)
961             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
962             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
963             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
964             "packuswb            %%mm7, %%mm1       \n\t"
965             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
966             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "r" (dest),
967                "a" (&c->redDither),
968                "r" (abuf0), "r" (abuf1)
969             : "%r8"
970         );
971 #else
972         *(const uint16_t **)(&c->u_temp)=abuf0;
973         *(const uint16_t **)(&c->v_temp)=abuf1;
974         __asm__ volatile(
975             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
976             "mov        %4, %%"REG_b"               \n\t"
977             "push %%"REG_BP"                        \n\t"
978             YSCALEYUV2RGB(%%REGBP, %5)
979             "push                   %0              \n\t"
980             "push                   %1              \n\t"
981             "mov          "U_TEMP"(%5), %0          \n\t"
982             "mov          "V_TEMP"(%5), %1          \n\t"
983             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
984             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
985             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
986             "packuswb            %%mm7, %%mm1       \n\t"
987             "pop                    %1              \n\t"
988             "pop                    %0              \n\t"
989             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
990             "pop %%"REG_BP"                         \n\t"
991             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
992             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
993                "a" (&c->redDither)
994         );
995 #endif
996     } else {
997         __asm__ volatile(
998             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
999             "mov        %4, %%"REG_b"               \n\t"
1000             "push %%"REG_BP"                        \n\t"
1001             YSCALEYUV2RGB(%%REGBP, %5)
1002             "pcmpeqd %%mm7, %%mm7                   \n\t"
1003             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1004             "pop %%"REG_BP"                         \n\t"
1005             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1006             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1007                "a" (&c->redDither)
1008         );
1009     }
1010 }
1011
1012 static inline void RENAME(yuv2bgr24_2)(SwsContext *c, const uint16_t *buf0,
1013                                        const uint16_t *buf1, const uint16_t *uvbuf0,
1014                                        const uint16_t *uvbuf1, const uint16_t *abuf0,
1015                                        const uint16_t *abuf1, uint8_t *dest,
1016                                        int dstW, int yalpha, int uvalpha, int y)
1017 {
1018     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1019     __asm__ volatile(
1020         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1021         "mov        %4, %%"REG_b"               \n\t"
1022         "push %%"REG_BP"                        \n\t"
1023         YSCALEYUV2RGB(%%REGBP, %5)
1024         "pxor    %%mm7, %%mm7                   \n\t"
1025         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1026         "pop %%"REG_BP"                         \n\t"
1027         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1028         :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1029            "a" (&c->redDither)
1030     );
1031 }
1032
1033 static inline void RENAME(yuv2rgb555_2)(SwsContext *c, const uint16_t *buf0,
1034                                         const uint16_t *buf1, const uint16_t *uvbuf0,
1035                                         const uint16_t *uvbuf1, const uint16_t *abuf0,
1036                                         const uint16_t *abuf1, uint8_t *dest,
1037                                         int dstW, int yalpha, int uvalpha, int y)
1038 {
1039     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1040     __asm__ volatile(
1041         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1042         "mov        %4, %%"REG_b"               \n\t"
1043         "push %%"REG_BP"                        \n\t"
1044         YSCALEYUV2RGB(%%REGBP, %5)
1045         "pxor    %%mm7, %%mm7                   \n\t"
1046         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1047 #ifdef DITHER1XBPP
1048         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1049         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1050         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1051 #endif
1052         WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1053         "pop %%"REG_BP"                         \n\t"
1054         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1055         :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1056            "a" (&c->redDither)
1057     );
1058 }
1059
1060 static inline void RENAME(yuv2rgb565_2)(SwsContext *c, const uint16_t *buf0,
1061                                         const uint16_t *buf1, const uint16_t *uvbuf0,
1062                                         const uint16_t *uvbuf1, const uint16_t *abuf0,
1063                                         const uint16_t *abuf1, uint8_t *dest,
1064                                         int dstW, int yalpha, int uvalpha, int y)
1065 {
1066     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1067     __asm__ volatile(
1068         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1069         "mov        %4, %%"REG_b"               \n\t"
1070         "push %%"REG_BP"                        \n\t"
1071         YSCALEYUV2RGB(%%REGBP, %5)
1072         "pxor    %%mm7, %%mm7                   \n\t"
1073         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1074 #ifdef DITHER1XBPP
1075         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1076         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1077         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1078 #endif
1079         WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1080         "pop %%"REG_BP"                         \n\t"
1081         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1082         :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1083            "a" (&c->redDither)
1084     );
1085 }
1086
1087 #define REAL_YSCALEYUV2PACKED(index, c) \
1088     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
1089     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
1090     "psraw                $3, %%mm0                           \n\t"\
1091     "psraw                $3, %%mm1                           \n\t"\
1092     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1093     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1094     "xor            "#index", "#index"                        \n\t"\
1095     ".p2align              4            \n\t"\
1096     "1:                                 \n\t"\
1097     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1098     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1099     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1100     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1101     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
1102     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
1103     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
1104     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
1105     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
1106     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1107     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1108     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
1109     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
1110     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
1111     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
1112     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
1113     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
1114     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
1115     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
1116     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1117     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1118     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1119     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1120     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1121     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1122
1123 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
1124
1125 static inline void RENAME(yuv2yuyv422_2)(SwsContext *c, const uint16_t *buf0,
1126                                          const uint16_t *buf1, const uint16_t *uvbuf0,
1127                                          const uint16_t *uvbuf1, const uint16_t *abuf0,
1128                                          const uint16_t *abuf1, uint8_t *dest,
1129                                          int dstW, int yalpha, int uvalpha, int y)
1130 {
1131     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1132     __asm__ volatile(
1133         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1134         "mov %4, %%"REG_b"                        \n\t"
1135         "push %%"REG_BP"                        \n\t"
1136         YSCALEYUV2PACKED(%%REGBP, %5)
1137         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1138         "pop %%"REG_BP"                         \n\t"
1139         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1140         :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1141            "a" (&c->redDither)
1142     );
1143 }
1144
1145 #define REAL_YSCALEYUV2RGB1(index, c) \
1146     "xor            "#index", "#index"  \n\t"\
1147     ".p2align              4            \n\t"\
1148     "1:                                 \n\t"\
1149     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1150     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1151     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1152     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1153     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1154     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1155     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1156     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1157     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1158     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1159     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1160     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1161     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1162     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1163     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1164     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1165     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1166     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1167     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1168     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1169     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1170     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1171     "paddw             %%mm3, %%mm4     \n\t"\
1172     "movq              %%mm2, %%mm0     \n\t"\
1173     "movq              %%mm5, %%mm6     \n\t"\
1174     "movq              %%mm4, %%mm3     \n\t"\
1175     "punpcklwd         %%mm2, %%mm2     \n\t"\
1176     "punpcklwd         %%mm5, %%mm5     \n\t"\
1177     "punpcklwd         %%mm4, %%mm4     \n\t"\
1178     "paddw             %%mm1, %%mm2     \n\t"\
1179     "paddw             %%mm1, %%mm5     \n\t"\
1180     "paddw             %%mm1, %%mm4     \n\t"\
1181     "punpckhwd         %%mm0, %%mm0     \n\t"\
1182     "punpckhwd         %%mm6, %%mm6     \n\t"\
1183     "punpckhwd         %%mm3, %%mm3     \n\t"\
1184     "paddw             %%mm7, %%mm0     \n\t"\
1185     "paddw             %%mm7, %%mm6     \n\t"\
1186     "paddw             %%mm7, %%mm3     \n\t"\
1187     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1188     "packuswb          %%mm0, %%mm2     \n\t"\
1189     "packuswb          %%mm6, %%mm5     \n\t"\
1190     "packuswb          %%mm3, %%mm4     \n\t"\
1191
1192 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
1193
1194 // do vertical chrominance interpolation
1195 #define REAL_YSCALEYUV2RGB1b(index, c) \
1196     "xor            "#index", "#index"  \n\t"\
1197     ".p2align              4            \n\t"\
1198     "1:                                 \n\t"\
1199     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1200     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1201     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1202     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1203     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1204     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1205     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
1206     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
1207     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1208     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1209     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1210     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1211     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1212     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1213     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1214     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1215     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1216     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1217     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1218     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1219     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1220     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1221     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1222     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1223     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1224     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1225     "paddw             %%mm3, %%mm4     \n\t"\
1226     "movq              %%mm2, %%mm0     \n\t"\
1227     "movq              %%mm5, %%mm6     \n\t"\
1228     "movq              %%mm4, %%mm3     \n\t"\
1229     "punpcklwd         %%mm2, %%mm2     \n\t"\
1230     "punpcklwd         %%mm5, %%mm5     \n\t"\
1231     "punpcklwd         %%mm4, %%mm4     \n\t"\
1232     "paddw             %%mm1, %%mm2     \n\t"\
1233     "paddw             %%mm1, %%mm5     \n\t"\
1234     "paddw             %%mm1, %%mm4     \n\t"\
1235     "punpckhwd         %%mm0, %%mm0     \n\t"\
1236     "punpckhwd         %%mm6, %%mm6     \n\t"\
1237     "punpckhwd         %%mm3, %%mm3     \n\t"\
1238     "paddw             %%mm7, %%mm0     \n\t"\
1239     "paddw             %%mm7, %%mm6     \n\t"\
1240     "paddw             %%mm7, %%mm3     \n\t"\
1241     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1242     "packuswb          %%mm0, %%mm2     \n\t"\
1243     "packuswb          %%mm6, %%mm5     \n\t"\
1244     "packuswb          %%mm3, %%mm4     \n\t"\
1245
1246 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
1247
1248 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
1249     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
1250     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
1251     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
1252     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
1253     "packuswb          %%mm1, %%mm7     \n\t"
1254 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
1255
1256 /**
1257  * YV12 to RGB without scaling or interpolating
1258  */
1259 static inline void RENAME(yuv2rgb32_1)(SwsContext *c, const uint16_t *buf0,
1260                                        const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1261                                        const uint16_t *abuf0, uint8_t *dest,
1262                                        int dstW, int uvalpha, enum PixelFormat dstFormat,
1263                                        int flags, int y)
1264 {
1265     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1266
1267     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1268         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1269             __asm__ volatile(
1270                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1271                 "mov        %4, %%"REG_b"               \n\t"
1272                 "push %%"REG_BP"                        \n\t"
1273                 YSCALEYUV2RGB1(%%REGBP, %5)
1274                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1275                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1276                 "pop %%"REG_BP"                         \n\t"
1277                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1278                 :: "c" (buf0), "d" (abuf0), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1279                    "a" (&c->redDither)
1280             );
1281         } else {
1282             __asm__ volatile(
1283                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1284                 "mov        %4, %%"REG_b"               \n\t"
1285                 "push %%"REG_BP"                        \n\t"
1286                 YSCALEYUV2RGB1(%%REGBP, %5)
1287                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1288                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1289                 "pop %%"REG_BP"                         \n\t"
1290                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1291                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1292                    "a" (&c->redDither)
1293             );
1294         }
1295     } else {
1296         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1297             __asm__ volatile(
1298                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1299                 "mov        %4, %%"REG_b"               \n\t"
1300                 "push %%"REG_BP"                        \n\t"
1301                 YSCALEYUV2RGB1b(%%REGBP, %5)
1302                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1303                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1304                 "pop %%"REG_BP"                         \n\t"
1305                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1306                 :: "c" (buf0), "d" (abuf0), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1307                    "a" (&c->redDither)
1308             );
1309         } else {
1310             __asm__ volatile(
1311                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1312                 "mov        %4, %%"REG_b"               \n\t"
1313                 "push %%"REG_BP"                        \n\t"
1314                 YSCALEYUV2RGB1b(%%REGBP, %5)
1315                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1316                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1317                 "pop %%"REG_BP"                         \n\t"
1318                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1319                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1320                    "a" (&c->redDither)
1321             );
1322         }
1323     }
1324 }
1325
1326 static inline void RENAME(yuv2bgr24_1)(SwsContext *c, const uint16_t *buf0,
1327                                        const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1328                                        const uint16_t *abuf0, uint8_t *dest,
1329                                        int dstW, int uvalpha, enum PixelFormat dstFormat,
1330                                        int flags, int y)
1331 {
1332     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1333
1334     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1335         __asm__ volatile(
1336             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1337             "mov        %4, %%"REG_b"               \n\t"
1338             "push %%"REG_BP"                        \n\t"
1339             YSCALEYUV2RGB1(%%REGBP, %5)
1340             "pxor    %%mm7, %%mm7                   \n\t"
1341             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1342             "pop %%"REG_BP"                         \n\t"
1343             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1344             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1345                "a" (&c->redDither)
1346         );
1347     } else {
1348         __asm__ volatile(
1349             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1350             "mov        %4, %%"REG_b"               \n\t"
1351             "push %%"REG_BP"                        \n\t"
1352             YSCALEYUV2RGB1b(%%REGBP, %5)
1353             "pxor    %%mm7, %%mm7                   \n\t"
1354             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1355             "pop %%"REG_BP"                         \n\t"
1356             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1357             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1358                "a" (&c->redDither)
1359         );
1360     }
1361 }
1362
1363 static inline void RENAME(yuv2rgb555_1)(SwsContext *c, const uint16_t *buf0,
1364                                         const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1365                                         const uint16_t *abuf0, uint8_t *dest,
1366                                         int dstW, int uvalpha, enum PixelFormat dstFormat,
1367                                         int flags, int y)
1368 {
1369     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1370
1371     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1372         __asm__ volatile(
1373             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1374             "mov        %4, %%"REG_b"               \n\t"
1375             "push %%"REG_BP"                        \n\t"
1376             YSCALEYUV2RGB1(%%REGBP, %5)
1377             "pxor    %%mm7, %%mm7                   \n\t"
1378             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1379 #ifdef DITHER1XBPP
1380             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1381             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1382             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1383 #endif
1384             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1385             "pop %%"REG_BP"                         \n\t"
1386             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1387             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1388                "a" (&c->redDither)
1389         );
1390     } else {
1391         __asm__ volatile(
1392             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1393             "mov        %4, %%"REG_b"               \n\t"
1394             "push %%"REG_BP"                        \n\t"
1395             YSCALEYUV2RGB1b(%%REGBP, %5)
1396             "pxor    %%mm7, %%mm7                   \n\t"
1397             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1398 #ifdef DITHER1XBPP
1399             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1400             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1401             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1402 #endif
1403             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1404             "pop %%"REG_BP"                         \n\t"
1405             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1406             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1407                "a" (&c->redDither)
1408         );
1409     }
1410 }
1411
1412 static inline void RENAME(yuv2rgb565_1)(SwsContext *c, const uint16_t *buf0,
1413                                         const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1414                                         const uint16_t *abuf0, uint8_t *dest,
1415                                         int dstW, int uvalpha, enum PixelFormat dstFormat,
1416                                         int flags, int y)
1417 {
1418     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1419
1420     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1421         __asm__ volatile(
1422             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1423             "mov        %4, %%"REG_b"               \n\t"
1424             "push %%"REG_BP"                        \n\t"
1425             YSCALEYUV2RGB1(%%REGBP, %5)
1426             "pxor    %%mm7, %%mm7                   \n\t"
1427             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1428 #ifdef DITHER1XBPP
1429             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1430             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1431             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1432 #endif
1433             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1434             "pop %%"REG_BP"                         \n\t"
1435             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1436             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1437                "a" (&c->redDither)
1438         );
1439     } else {
1440         __asm__ volatile(
1441             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1442             "mov        %4, %%"REG_b"               \n\t"
1443             "push %%"REG_BP"                        \n\t"
1444             YSCALEYUV2RGB1b(%%REGBP, %5)
1445             "pxor    %%mm7, %%mm7                   \n\t"
1446             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1447 #ifdef DITHER1XBPP
1448             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1449             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1450             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1451 #endif
1452             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1453             "pop %%"REG_BP"                         \n\t"
1454             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1455             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1456                "a" (&c->redDither)
1457         );
1458     }
1459 }
1460
1461 #define REAL_YSCALEYUV2PACKED1(index, c) \
1462     "xor            "#index", "#index"  \n\t"\
1463     ".p2align              4            \n\t"\
1464     "1:                                 \n\t"\
1465     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1466     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1467     "psraw                $7, %%mm3     \n\t" \
1468     "psraw                $7, %%mm4     \n\t" \
1469     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1470     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1471     "psraw                $7, %%mm1     \n\t" \
1472     "psraw                $7, %%mm7     \n\t" \
1473
1474 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
1475
1476 #define REAL_YSCALEYUV2PACKED1b(index, c) \
1477     "xor "#index", "#index"             \n\t"\
1478     ".p2align              4            \n\t"\
1479     "1:                                 \n\t"\
1480     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1481     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1482     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1483     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1484     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1485     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1486     "psrlw                $8, %%mm3     \n\t" \
1487     "psrlw                $8, %%mm4     \n\t" \
1488     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1489     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1490     "psraw                $7, %%mm1     \n\t" \
1491     "psraw                $7, %%mm7     \n\t"
1492 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
1493
1494 static inline void RENAME(yuv2yuyv422_1)(SwsContext *c, const uint16_t *buf0,
1495                                          const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1496                                          const uint16_t *abuf0, uint8_t *dest,
1497                                          int dstW, int uvalpha, enum PixelFormat dstFormat,
1498                                          int flags, int y)
1499 {
1500     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1501
1502     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1503         __asm__ volatile(
1504             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1505             "mov        %4, %%"REG_b"               \n\t"
1506             "push %%"REG_BP"                        \n\t"
1507             YSCALEYUV2PACKED1(%%REGBP, %5)
1508             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1509             "pop %%"REG_BP"                         \n\t"
1510             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1511             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1512                "a" (&c->redDither)
1513         );
1514     } else {
1515         __asm__ volatile(
1516             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1517             "mov        %4, %%"REG_b"               \n\t"
1518             "push %%"REG_BP"                        \n\t"
1519             YSCALEYUV2PACKED1b(%%REGBP, %5)
1520             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1521             "pop %%"REG_BP"                         \n\t"
1522             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1523             :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1524                "a" (&c->redDither)
1525         );
1526     }
1527 }
1528
1529 #if !COMPILE_TEMPLATE_MMX2
1530 //FIXME yuy2* can read up to 7 samples too much
1531
1532 static inline void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1533 {
1534     __asm__ volatile(
1535         "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1536         "mov                    %0, %%"REG_a"       \n\t"
1537         "1:                                         \n\t"
1538         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1539         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1540         "pand                %%mm2, %%mm0           \n\t"
1541         "pand                %%mm2, %%mm1           \n\t"
1542         "packuswb            %%mm1, %%mm0           \n\t"
1543         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1544         "add                    $8, %%"REG_a"       \n\t"
1545         " js                    1b                  \n\t"
1546         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1547         : "%"REG_a
1548     );
1549 }
1550
1551 static inline void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1552 {
1553     __asm__ volatile(
1554         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1555         "mov                    %0, %%"REG_a"       \n\t"
1556         "1:                                         \n\t"
1557         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1558         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1559         "psrlw                  $8, %%mm0           \n\t"
1560         "psrlw                  $8, %%mm1           \n\t"
1561         "packuswb            %%mm1, %%mm0           \n\t"
1562         "movq                %%mm0, %%mm1           \n\t"
1563         "psrlw                  $8, %%mm0           \n\t"
1564         "pand                %%mm4, %%mm1           \n\t"
1565         "packuswb            %%mm0, %%mm0           \n\t"
1566         "packuswb            %%mm1, %%mm1           \n\t"
1567         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1568         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1569         "add                    $4, %%"REG_a"       \n\t"
1570         " js                    1b                  \n\t"
1571         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1572         : "%"REG_a
1573     );
1574     assert(src1 == src2);
1575 }
1576
1577 static inline void RENAME(LEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1578 {
1579     __asm__ volatile(
1580         "mov                    %0, %%"REG_a"       \n\t"
1581         "1:                                         \n\t"
1582         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1583         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1584         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1585         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1586         "psrlw                  $8, %%mm0           \n\t"
1587         "psrlw                  $8, %%mm1           \n\t"
1588         "psrlw                  $8, %%mm2           \n\t"
1589         "psrlw                  $8, %%mm3           \n\t"
1590         "packuswb            %%mm1, %%mm0           \n\t"
1591         "packuswb            %%mm3, %%mm2           \n\t"
1592         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1593         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1594         "add                    $8, %%"REG_a"       \n\t"
1595         " js                    1b                  \n\t"
1596         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1597         : "%"REG_a
1598     );
1599 }
1600
1601 /* This is almost identical to the previous, end exists only because
1602  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1603 static inline void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1604 {
1605     __asm__ volatile(
1606         "mov                  %0, %%"REG_a"         \n\t"
1607         "1:                                         \n\t"
1608         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1609         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1610         "psrlw                $8, %%mm0             \n\t"
1611         "psrlw                $8, %%mm1             \n\t"
1612         "packuswb          %%mm1, %%mm0             \n\t"
1613         "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1614         "add                  $8, %%"REG_a"         \n\t"
1615         " js                  1b                    \n\t"
1616         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1617         : "%"REG_a
1618     );
1619 }
1620
1621 static inline void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1622 {
1623     __asm__ volatile(
1624         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1625         "mov                    %0, %%"REG_a"       \n\t"
1626         "1:                                         \n\t"
1627         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1628         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1629         "pand                %%mm4, %%mm0           \n\t"
1630         "pand                %%mm4, %%mm1           \n\t"
1631         "packuswb            %%mm1, %%mm0           \n\t"
1632         "movq                %%mm0, %%mm1           \n\t"
1633         "psrlw                  $8, %%mm0           \n\t"
1634         "pand                %%mm4, %%mm1           \n\t"
1635         "packuswb            %%mm0, %%mm0           \n\t"
1636         "packuswb            %%mm1, %%mm1           \n\t"
1637         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1638         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1639         "add                    $4, %%"REG_a"       \n\t"
1640         " js                    1b                  \n\t"
1641         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1642         : "%"REG_a
1643     );
1644     assert(src1 == src2);
1645 }
1646
1647 static inline void RENAME(BEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1648 {
1649     __asm__ volatile(
1650         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1651         "mov                    %0, %%"REG_a"       \n\t"
1652         "1:                                         \n\t"
1653         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1654         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1655         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1656         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1657         "pand                %%mm4, %%mm0           \n\t"
1658         "pand                %%mm4, %%mm1           \n\t"
1659         "pand                %%mm4, %%mm2           \n\t"
1660         "pand                %%mm4, %%mm3           \n\t"
1661         "packuswb            %%mm1, %%mm0           \n\t"
1662         "packuswb            %%mm3, %%mm2           \n\t"
1663         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1664         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1665         "add                    $8, %%"REG_a"       \n\t"
1666         " js                    1b                  \n\t"
1667         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1668         : "%"REG_a
1669     );
1670 }
1671
1672 static inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1673                                     const uint8_t *src, long width)
1674 {
1675     __asm__ volatile(
1676         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1677         "mov                    %0, %%"REG_a"       \n\t"
1678         "1:                                         \n\t"
1679         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1680         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1681         "movq                %%mm0, %%mm2           \n\t"
1682         "movq                %%mm1, %%mm3           \n\t"
1683         "pand                %%mm4, %%mm0           \n\t"
1684         "pand                %%mm4, %%mm1           \n\t"
1685         "psrlw                  $8, %%mm2           \n\t"
1686         "psrlw                  $8, %%mm3           \n\t"
1687         "packuswb            %%mm1, %%mm0           \n\t"
1688         "packuswb            %%mm3, %%mm2           \n\t"
1689         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1690         "movq                %%mm2, (%3, %%"REG_a") \n\t"
1691         "add                    $8, %%"REG_a"       \n\t"
1692         " js                    1b                  \n\t"
1693         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1694         : "%"REG_a
1695     );
1696 }
1697
1698 static inline void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1699                                     const uint8_t *src1, const uint8_t *src2,
1700                                     long width, uint32_t *unused)
1701 {
1702     RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1703 }
1704
1705 static inline void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1706                                     const uint8_t *src1, const uint8_t *src2,
1707                                     long width, uint32_t *unused)
1708 {
1709     RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1710 }
1711 #endif /* !COMPILE_TEMPLATE_MMX2 */
1712
1713 static inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, const uint8_t *src, long width, enum PixelFormat srcFormat)
1714 {
1715
1716     if(srcFormat == PIX_FMT_BGR24) {
1717         __asm__ volatile(
1718             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1719             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1720             :
1721         );
1722     } else {
1723         __asm__ volatile(
1724             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1725             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1726             :
1727         );
1728     }
1729
1730     __asm__ volatile(
1731         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1732         "mov                        %2, %%"REG_a"   \n\t"
1733         "pxor                    %%mm7, %%mm7       \n\t"
1734         "1:                                         \n\t"
1735         PREFETCH"               64(%0)              \n\t"
1736         "movd                     (%0), %%mm0       \n\t"
1737         "movd                    2(%0), %%mm1       \n\t"
1738         "movd                    6(%0), %%mm2       \n\t"
1739         "movd                    8(%0), %%mm3       \n\t"
1740         "add                       $12, %0          \n\t"
1741         "punpcklbw               %%mm7, %%mm0       \n\t"
1742         "punpcklbw               %%mm7, %%mm1       \n\t"
1743         "punpcklbw               %%mm7, %%mm2       \n\t"
1744         "punpcklbw               %%mm7, %%mm3       \n\t"
1745         "pmaddwd                 %%mm5, %%mm0       \n\t"
1746         "pmaddwd                 %%mm6, %%mm1       \n\t"
1747         "pmaddwd                 %%mm5, %%mm2       \n\t"
1748         "pmaddwd                 %%mm6, %%mm3       \n\t"
1749         "paddd                   %%mm1, %%mm0       \n\t"
1750         "paddd                   %%mm3, %%mm2       \n\t"
1751         "paddd                   %%mm4, %%mm0       \n\t"
1752         "paddd                   %%mm4, %%mm2       \n\t"
1753         "psrad                     $15, %%mm0       \n\t"
1754         "psrad                     $15, %%mm2       \n\t"
1755         "packssdw                %%mm2, %%mm0       \n\t"
1756         "packuswb                %%mm0, %%mm0       \n\t"
1757         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1758         "add                        $4, %%"REG_a"   \n\t"
1759         " js                        1b              \n\t"
1760     : "+r" (src)
1761     : "r" (dst+width), "g" ((x86_reg)-width)
1762     : "%"REG_a
1763     );
1764 }
1765
1766 static inline void RENAME(bgr24ToUV_mmx)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src, long width, enum PixelFormat srcFormat)
1767 {
1768     __asm__ volatile(
1769         "movq                    24(%4), %%mm6       \n\t"
1770         "mov                        %3, %%"REG_a"   \n\t"
1771         "pxor                    %%mm7, %%mm7       \n\t"
1772         "1:                                         \n\t"
1773         PREFETCH"               64(%0)              \n\t"
1774         "movd                     (%0), %%mm0       \n\t"
1775         "movd                    2(%0), %%mm1       \n\t"
1776         "punpcklbw               %%mm7, %%mm0       \n\t"
1777         "punpcklbw               %%mm7, %%mm1       \n\t"
1778         "movq                    %%mm0, %%mm2       \n\t"
1779         "movq                    %%mm1, %%mm3       \n\t"
1780         "pmaddwd                  (%4), %%mm0       \n\t"
1781         "pmaddwd                 8(%4), %%mm1       \n\t"
1782         "pmaddwd                16(%4), %%mm2       \n\t"
1783         "pmaddwd                 %%mm6, %%mm3       \n\t"
1784         "paddd                   %%mm1, %%mm0       \n\t"
1785         "paddd                   %%mm3, %%mm2       \n\t"
1786
1787         "movd                    6(%0), %%mm1       \n\t"
1788         "movd                    8(%0), %%mm3       \n\t"
1789         "add                       $12, %0          \n\t"
1790         "punpcklbw               %%mm7, %%mm1       \n\t"
1791         "punpcklbw               %%mm7, %%mm3       \n\t"
1792         "movq                    %%mm1, %%mm4       \n\t"
1793         "movq                    %%mm3, %%mm5       \n\t"
1794         "pmaddwd                  (%4), %%mm1       \n\t"
1795         "pmaddwd                 8(%4), %%mm3       \n\t"
1796         "pmaddwd                16(%4), %%mm4       \n\t"
1797         "pmaddwd                 %%mm6, %%mm5       \n\t"
1798         "paddd                   %%mm3, %%mm1       \n\t"
1799         "paddd                   %%mm5, %%mm4       \n\t"
1800
1801         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1802         "paddd                   %%mm3, %%mm0       \n\t"
1803         "paddd                   %%mm3, %%mm2       \n\t"
1804         "paddd                   %%mm3, %%mm1       \n\t"
1805         "paddd                   %%mm3, %%mm4       \n\t"
1806         "psrad                     $15, %%mm0       \n\t"
1807         "psrad                     $15, %%mm2       \n\t"
1808         "psrad                     $15, %%mm1       \n\t"
1809         "psrad                     $15, %%mm4       \n\t"
1810         "packssdw                %%mm1, %%mm0       \n\t"
1811         "packssdw                %%mm4, %%mm2       \n\t"
1812         "packuswb                %%mm0, %%mm0       \n\t"
1813         "packuswb                %%mm2, %%mm2       \n\t"
1814         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1815         "movd                %%mm2, (%2, %%"REG_a") \n\t"
1816         "add                        $4, %%"REG_a"   \n\t"
1817         " js                        1b              \n\t"
1818     : "+r" (src)
1819     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1820     : "%"REG_a
1821     );
1822 }
1823
1824 static inline void RENAME(bgr24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1825 {
1826     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1827 }
1828
1829 static inline void RENAME(bgr24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1830 {
1831     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1832     assert(src1 == src2);
1833 }
1834
1835 static inline void RENAME(rgb24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1836 {
1837     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1838 }
1839
1840 static inline void RENAME(rgb24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1841 {
1842     assert(src1==src2);
1843     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1844 }
1845
1846 #if !COMPILE_TEMPLATE_MMX2
1847 // bilinear / bicubic scaling
1848 static inline void RENAME(hScale)(int16_t *dst, int dstW, const uint8_t *src, int srcW, int xInc,
1849                                   const int16_t *filter, const int16_t *filterPos, long filterSize)
1850 {
1851     assert(filterSize % 4 == 0 && filterSize>0);
1852     if (filterSize==4) { // Always true for upscaling, sometimes for down, too.
1853         x86_reg counter= -2*dstW;
1854         filter-= counter*2;
1855         filterPos-= counter/2;
1856         dst-= counter/2;
1857         __asm__ volatile(
1858 #if defined(PIC)
1859             "push            %%"REG_b"              \n\t"
1860 #endif
1861             "pxor                %%mm7, %%mm7       \n\t"
1862             "push           %%"REG_BP"              \n\t" // we use 7 regs here ...
1863             "mov             %%"REG_a", %%"REG_BP"  \n\t"
1864             ".p2align                4              \n\t"
1865             "1:                                     \n\t"
1866             "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"
1867             "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"
1868             "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"
1869             "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"
1870             "movd      (%3, %%"REG_a"), %%mm0       \n\t"
1871             "movd      (%3, %%"REG_b"), %%mm2       \n\t"
1872             "punpcklbw           %%mm7, %%mm0       \n\t"
1873             "punpcklbw           %%mm7, %%mm2       \n\t"
1874             "pmaddwd             %%mm1, %%mm0       \n\t"
1875             "pmaddwd             %%mm2, %%mm3       \n\t"
1876             "movq                %%mm0, %%mm4       \n\t"
1877             "punpckldq           %%mm3, %%mm0       \n\t"
1878             "punpckhdq           %%mm3, %%mm4       \n\t"
1879             "paddd               %%mm4, %%mm0       \n\t"
1880             "psrad                  $7, %%mm0       \n\t"
1881             "packssdw            %%mm0, %%mm0       \n\t"
1882             "movd                %%mm0, (%4, %%"REG_BP")    \n\t"
1883             "add                    $4, %%"REG_BP"  \n\t"
1884             " jnc                   1b              \n\t"
1885
1886             "pop            %%"REG_BP"              \n\t"
1887 #if defined(PIC)
1888             "pop             %%"REG_b"              \n\t"
1889 #endif
1890             : "+a" (counter)
1891             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1892 #if !defined(PIC)
1893             : "%"REG_b
1894 #endif
1895         );
1896     } else if (filterSize==8) {
1897         x86_reg counter= -2*dstW;
1898         filter-= counter*4;
1899         filterPos-= counter/2;
1900         dst-= counter/2;
1901         __asm__ volatile(
1902 #if defined(PIC)
1903             "push             %%"REG_b"             \n\t"
1904 #endif
1905             "pxor                 %%mm7, %%mm7      \n\t"
1906             "push            %%"REG_BP"             \n\t" // we use 7 regs here ...
1907             "mov              %%"REG_a", %%"REG_BP" \n\t"
1908             ".p2align                 4             \n\t"
1909             "1:                                     \n\t"
1910             "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"
1911             "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"
1912             "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"
1913             "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"
1914             "movd       (%3, %%"REG_a"), %%mm0      \n\t"
1915             "movd       (%3, %%"REG_b"), %%mm2      \n\t"
1916             "punpcklbw            %%mm7, %%mm0      \n\t"
1917             "punpcklbw            %%mm7, %%mm2      \n\t"
1918             "pmaddwd              %%mm1, %%mm0      \n\t"
1919             "pmaddwd              %%mm2, %%mm3      \n\t"
1920
1921             "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"
1922             "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"
1923             "movd      4(%3, %%"REG_a"), %%mm4      \n\t"
1924             "movd      4(%3, %%"REG_b"), %%mm2      \n\t"
1925             "punpcklbw            %%mm7, %%mm4      \n\t"
1926             "punpcklbw            %%mm7, %%mm2      \n\t"
1927             "pmaddwd              %%mm1, %%mm4      \n\t"
1928             "pmaddwd              %%mm2, %%mm5      \n\t"
1929             "paddd                %%mm4, %%mm0      \n\t"
1930             "paddd                %%mm5, %%mm3      \n\t"
1931             "movq                 %%mm0, %%mm4      \n\t"
1932             "punpckldq            %%mm3, %%mm0      \n\t"
1933             "punpckhdq            %%mm3, %%mm4      \n\t"
1934             "paddd                %%mm4, %%mm0      \n\t"
1935             "psrad                   $7, %%mm0      \n\t"
1936             "packssdw             %%mm0, %%mm0      \n\t"
1937             "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"
1938             "add                     $4, %%"REG_BP" \n\t"
1939             " jnc                    1b             \n\t"
1940
1941             "pop             %%"REG_BP"             \n\t"
1942 #if defined(PIC)
1943             "pop              %%"REG_b"             \n\t"
1944 #endif
1945             : "+a" (counter)
1946             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1947 #if !defined(PIC)
1948             : "%"REG_b
1949 #endif
1950         );
1951     } else {
1952         const uint8_t *offset = src+filterSize;
1953         x86_reg counter= -2*dstW;
1954         //filter-= counter*filterSize/2;
1955         filterPos-= counter/2;
1956         dst-= counter/2;
1957         __asm__ volatile(
1958             "pxor                  %%mm7, %%mm7     \n\t"
1959             ".p2align                  4            \n\t"
1960             "1:                                     \n\t"
1961             "mov                      %2, %%"REG_c" \n\t"
1962             "movzwl      (%%"REG_c", %0), %%eax     \n\t"
1963             "movzwl     2(%%"REG_c", %0), %%edx     \n\t"
1964             "mov                      %5, %%"REG_c" \n\t"
1965             "pxor                  %%mm4, %%mm4     \n\t"
1966             "pxor                  %%mm5, %%mm5     \n\t"
1967             "2:                                     \n\t"
1968             "movq                   (%1), %%mm1     \n\t"
1969             "movq               (%1, %6), %%mm3     \n\t"
1970             "movd (%%"REG_c", %%"REG_a"), %%mm0     \n\t"
1971             "movd (%%"REG_c", %%"REG_d"), %%mm2     \n\t"
1972             "punpcklbw             %%mm7, %%mm0     \n\t"
1973             "punpcklbw             %%mm7, %%mm2     \n\t"
1974             "pmaddwd               %%mm1, %%mm0     \n\t"
1975             "pmaddwd               %%mm2, %%mm3     \n\t"
1976             "paddd                 %%mm3, %%mm5     \n\t"
1977             "paddd                 %%mm0, %%mm4     \n\t"
1978             "add                      $8, %1        \n\t"
1979             "add                      $4, %%"REG_c" \n\t"
1980             "cmp                      %4, %%"REG_c" \n\t"
1981             " jb                      2b            \n\t"
1982             "add                      %6, %1        \n\t"
1983             "movq                  %%mm4, %%mm0     \n\t"
1984             "punpckldq             %%mm5, %%mm4     \n\t"
1985             "punpckhdq             %%mm5, %%mm0     \n\t"
1986             "paddd                 %%mm0, %%mm4     \n\t"
1987             "psrad                    $7, %%mm4     \n\t"
1988             "packssdw              %%mm4, %%mm4     \n\t"
1989             "mov                      %3, %%"REG_a" \n\t"
1990             "movd                  %%mm4, (%%"REG_a", %0)   \n\t"
1991             "add                      $4, %0        \n\t"
1992             " jnc                     1b            \n\t"
1993
1994             : "+r" (counter), "+r" (filter)
1995             : "m" (filterPos), "m" (dst), "m"(offset),
1996             "m" (src), "r" ((x86_reg)filterSize*2)
1997             : "%"REG_a, "%"REG_c, "%"REG_d
1998         );
1999     }
2000 }
2001 #endif /* !COMPILE_TEMPLATE_MMX2 */
2002
2003 #if COMPILE_TEMPLATE_MMX2
2004 static inline void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
2005                                         long dstWidth, const uint8_t *src, int srcW,
2006                                         int xInc)
2007 {
2008     int32_t *filterPos = c->hLumFilterPos;
2009     int16_t *filter    = c->hLumFilter;
2010     int     canMMX2BeUsed  = c->canMMX2BeUsed;
2011     void    *mmx2FilterCode= c->lumMmx2FilterCode;
2012     int i;
2013 #if defined(PIC)
2014     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2015 #endif
2016
2017         __asm__ volatile(
2018 #if defined(PIC)
2019             "mov               %%"REG_b", %5        \n\t"
2020 #endif
2021             "pxor                  %%mm7, %%mm7     \n\t"
2022             "mov                      %0, %%"REG_c" \n\t"
2023             "mov                      %1, %%"REG_D" \n\t"
2024             "mov                      %2, %%"REG_d" \n\t"
2025             "mov                      %3, %%"REG_b" \n\t"
2026             "xor               %%"REG_a", %%"REG_a" \n\t" // i
2027             PREFETCH"        (%%"REG_c")            \n\t"
2028             PREFETCH"      32(%%"REG_c")            \n\t"
2029             PREFETCH"      64(%%"REG_c")            \n\t"
2030
2031 #if ARCH_X86_64
2032
2033 #define CALL_MMX2_FILTER_CODE \
2034             "movl            (%%"REG_b"), %%esi     \n\t"\
2035             "call                    *%4            \n\t"\
2036             "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
2037             "add               %%"REG_S", %%"REG_c" \n\t"\
2038             "add               %%"REG_a", %%"REG_D" \n\t"\
2039             "xor               %%"REG_a", %%"REG_a" \n\t"\
2040
2041 #else
2042
2043 #define CALL_MMX2_FILTER_CODE \
2044             "movl (%%"REG_b"), %%esi        \n\t"\
2045             "call         *%4                       \n\t"\
2046             "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
2047             "add               %%"REG_a", %%"REG_D" \n\t"\
2048             "xor               %%"REG_a", %%"REG_a" \n\t"\
2049
2050 #endif /* ARCH_X86_64 */
2051
2052             CALL_MMX2_FILTER_CODE
2053             CALL_MMX2_FILTER_CODE
2054             CALL_MMX2_FILTER_CODE
2055             CALL_MMX2_FILTER_CODE
2056             CALL_MMX2_FILTER_CODE
2057             CALL_MMX2_FILTER_CODE
2058             CALL_MMX2_FILTER_CODE
2059             CALL_MMX2_FILTER_CODE
2060
2061 #if defined(PIC)
2062             "mov                      %5, %%"REG_b" \n\t"
2063 #endif
2064             :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
2065             "m" (mmx2FilterCode)
2066 #if defined(PIC)
2067             ,"m" (ebxsave)
2068 #endif
2069             : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2070 #if !defined(PIC)
2071             ,"%"REG_b
2072 #endif
2073         );
2074         for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) dst[i] = src[srcW-1]*128;
2075 }
2076
2077 static inline void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst,
2078                                         long dstWidth, const uint8_t *src1,
2079                                         const uint8_t *src2, int srcW, int xInc)
2080 {
2081     int32_t *filterPos = c->hChrFilterPos;
2082     int16_t *filter    = c->hChrFilter;
2083     int     canMMX2BeUsed  = c->canMMX2BeUsed;
2084     void    *mmx2FilterCode= c->chrMmx2FilterCode;
2085     int i;
2086 #if defined(PIC)
2087     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2088 #endif
2089
2090         __asm__ volatile(
2091 #if defined(PIC)
2092             "mov          %%"REG_b", %6         \n\t"
2093 #endif
2094             "pxor             %%mm7, %%mm7      \n\t"
2095             "mov                 %0, %%"REG_c"  \n\t"
2096             "mov                 %1, %%"REG_D"  \n\t"
2097             "mov                 %2, %%"REG_d"  \n\t"
2098             "mov                 %3, %%"REG_b"  \n\t"
2099             "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2100             PREFETCH"   (%%"REG_c")             \n\t"
2101             PREFETCH" 32(%%"REG_c")             \n\t"
2102             PREFETCH" 64(%%"REG_c")             \n\t"
2103
2104             CALL_MMX2_FILTER_CODE
2105             CALL_MMX2_FILTER_CODE
2106             CALL_MMX2_FILTER_CODE
2107             CALL_MMX2_FILTER_CODE
2108             "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2109             "mov                 %5, %%"REG_c"  \n\t" // src
2110             "mov                 %1, %%"REG_D"  \n\t" // buf1
2111             "add              $"AV_STRINGIFY(VOF)", %%"REG_D"  \n\t"
2112             PREFETCH"   (%%"REG_c")             \n\t"
2113             PREFETCH" 32(%%"REG_c")             \n\t"
2114             PREFETCH" 64(%%"REG_c")             \n\t"
2115
2116             CALL_MMX2_FILTER_CODE
2117             CALL_MMX2_FILTER_CODE
2118             CALL_MMX2_FILTER_CODE
2119             CALL_MMX2_FILTER_CODE
2120
2121 #if defined(PIC)
2122             "mov %6, %%"REG_b"    \n\t"
2123 #endif
2124             :: "m" (src1), "m" (dst), "m" (filter), "m" (filterPos),
2125             "m" (mmx2FilterCode), "m" (src2)
2126 #if defined(PIC)
2127             ,"m" (ebxsave)
2128 #endif
2129             : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2130 #if !defined(PIC)
2131             ,"%"REG_b
2132 #endif
2133         );
2134         for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
2135             dst[i] = src1[srcW-1]*128;
2136             dst[i+VOFW] = src2[srcW-1]*128;
2137         }
2138 }
2139 #endif /* COMPILE_TEMPLATE_MMX2 */
2140
2141 #if !COMPILE_TEMPLATE_MMX2
2142 static void updateMMXDitherTables(SwsContext *c, int dstY, int lumBufIndex, int chrBufIndex,
2143                                   int lastInLumBuf, int lastInChrBuf)
2144 {
2145     const int dstH= c->dstH;
2146     const int flags= c->flags;
2147     int16_t **lumPixBuf= c->lumPixBuf;
2148     int16_t **chrPixBuf= c->chrPixBuf;
2149     int16_t **alpPixBuf= c->alpPixBuf;
2150     const int vLumBufSize= c->vLumBufSize;
2151     const int vChrBufSize= c->vChrBufSize;
2152     int16_t *vLumFilterPos= c->vLumFilterPos;
2153     int16_t *vChrFilterPos= c->vChrFilterPos;
2154     int16_t *vLumFilter= c->vLumFilter;
2155     int16_t *vChrFilter= c->vChrFilter;
2156     int32_t *lumMmxFilter= c->lumMmxFilter;
2157     int32_t *chrMmxFilter= c->chrMmxFilter;
2158     int32_t av_unused *alpMmxFilter= c->alpMmxFilter;
2159     const int vLumFilterSize= c->vLumFilterSize;
2160     const int vChrFilterSize= c->vChrFilterSize;
2161     const int chrDstY= dstY>>c->chrDstVSubSample;
2162     const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input
2163     const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input
2164
2165         c->blueDither= ff_dither8[dstY&1];
2166         if (c->dstFormat == PIX_FMT_RGB555 || c->dstFormat == PIX_FMT_BGR555)
2167             c->greenDither= ff_dither8[dstY&1];
2168         else
2169             c->greenDither= ff_dither4[dstY&1];
2170         c->redDither= ff_dither8[(dstY+1)&1];
2171         if (dstY < dstH - 2) {
2172             const int16_t **lumSrcPtr= (const int16_t **) lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2173             const int16_t **chrSrcPtr= (const int16_t **) chrPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2174             const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;
2175             int i;
2176             if (flags & SWS_ACCURATE_RND) {
2177                 int s= APCK_SIZE / 8;
2178                 for (i=0; i<vLumFilterSize; i+=2) {
2179                     *(const void**)&lumMmxFilter[s*i              ]= lumSrcPtr[i  ];
2180                     *(const void**)&lumMmxFilter[s*i+APCK_PTR2/4  ]= lumSrcPtr[i+(vLumFilterSize>1)];
2181                               lumMmxFilter[s*i+APCK_COEF/4  ]=
2182                               lumMmxFilter[s*i+APCK_COEF/4+1]= vLumFilter[dstY*vLumFilterSize + i    ]
2183                         + (vLumFilterSize>1 ? vLumFilter[dstY*vLumFilterSize + i + 1]<<16 : 0);
2184                     if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2185                         *(const void**)&alpMmxFilter[s*i              ]= alpSrcPtr[i  ];
2186                         *(const void**)&alpMmxFilter[s*i+APCK_PTR2/4  ]= alpSrcPtr[i+(vLumFilterSize>1)];
2187                                   alpMmxFilter[s*i+APCK_COEF/4  ]=
2188                                   alpMmxFilter[s*i+APCK_COEF/4+1]= lumMmxFilter[s*i+APCK_COEF/4  ];
2189                     }
2190                 }
2191                 for (i=0; i<vChrFilterSize; i+=2) {
2192                     *(const void**)&chrMmxFilter[s*i              ]= chrSrcPtr[i  ];
2193                     *(const void**)&chrMmxFilter[s*i+APCK_PTR2/4  ]= chrSrcPtr[i+(vChrFilterSize>1)];
2194                               chrMmxFilter[s*i+APCK_COEF/4  ]=
2195                               chrMmxFilter[s*i+APCK_COEF/4+1]= vChrFilter[chrDstY*vChrFilterSize + i    ]
2196                         + (vChrFilterSize>1 ? vChrFilter[chrDstY*vChrFilterSize + i + 1]<<16 : 0);
2197                 }
2198             } else {
2199                 for (i=0; i<vLumFilterSize; i++) {
2200                     lumMmxFilter[4*i+0]= (int32_t)lumSrcPtr[i];
2201                     lumMmxFilter[4*i+1]= (uint64_t)lumSrcPtr[i] >> 32;
2202                     lumMmxFilter[4*i+2]=
2203                     lumMmxFilter[4*i+3]=
2204                         ((uint16_t)vLumFilter[dstY*vLumFilterSize + i])*0x10001;
2205                     if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2206                         alpMmxFilter[4*i+0]= (int32_t)alpSrcPtr[i];
2207                         alpMmxFilter[4*i+1]= (uint64_t)alpSrcPtr[i] >> 32;
2208                         alpMmxFilter[4*i+2]=
2209                         alpMmxFilter[4*i+3]= lumMmxFilter[4*i+2];
2210                     }
2211                 }
2212                 for (i=0; i<vChrFilterSize; i++) {
2213                     chrMmxFilter[4*i+0]= (int32_t)chrSrcPtr[i];
2214                     chrMmxFilter[4*i+1]= (uint64_t)chrSrcPtr[i] >> 32;
2215                     chrMmxFilter[4*i+2]=
2216                     chrMmxFilter[4*i+3]=
2217                         ((uint16_t)vChrFilter[chrDstY*vChrFilterSize + i])*0x10001;
2218                 }
2219             }
2220         }
2221 }
2222 #endif /* !COMPILE_TEMPLATE_MMX2 */
2223
2224 static void RENAME(sws_init_swScale)(SwsContext *c)
2225 {
2226     enum PixelFormat srcFormat = c->srcFormat;
2227
2228     if (!(c->flags & SWS_BITEXACT)) {
2229         if (c->flags & SWS_ACCURATE_RND) {
2230             c->yuv2yuv1     = RENAME(yuv2yuv1_ar    );
2231             c->yuv2yuvX     = RENAME(yuv2yuvX_ar    );
2232             switch (c->dstFormat) {
2233             case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;
2234             case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;
2235             case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;
2236             case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;
2237             case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;
2238             default: break;
2239             }
2240         } else {
2241             c->yuv2yuv1     = RENAME(yuv2yuv1    );
2242             c->yuv2yuvX     = RENAME(yuv2yuvX    );
2243             switch (c->dstFormat) {
2244             case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;
2245             case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;
2246             case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;
2247             case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;
2248             case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;
2249             default: break;
2250             }
2251         }
2252         switch (c->dstFormat) {
2253         case PIX_FMT_RGB32:
2254                 c->yuv2packed1 = RENAME(yuv2rgb32_1);
2255                 c->yuv2packed2 = RENAME(yuv2rgb32_2);
2256                 break;
2257         case PIX_FMT_BGR24:
2258                 c->yuv2packed1 = RENAME(yuv2bgr24_1);
2259                 c->yuv2packed2 = RENAME(yuv2bgr24_2);
2260                 break;
2261         case PIX_FMT_RGB555:
2262                 c->yuv2packed1 = RENAME(yuv2rgb555_1);
2263                 c->yuv2packed2 = RENAME(yuv2rgb555_2);
2264                 break;
2265         case PIX_FMT_RGB565:
2266                 c->yuv2packed1 = RENAME(yuv2rgb565_1);
2267                 c->yuv2packed2 = RENAME(yuv2rgb565_2);
2268                 break;
2269         case PIX_FMT_YUYV422:
2270                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);
2271                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);
2272                 break;
2273         default:
2274                 break;
2275         }
2276     }
2277
2278 #if !COMPILE_TEMPLATE_MMX2
2279     c->hScale       = RENAME(hScale      );
2280 #endif /* !COMPILE_TEMPLATE_MMX2 */
2281
2282     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2283 #if COMPILE_TEMPLATE_MMX2
2284     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
2285     {
2286         c->hyscale_fast = RENAME(hyscale_fast);
2287         c->hcscale_fast = RENAME(hcscale_fast);
2288     } else {
2289 #endif /* COMPILE_TEMPLATE_MMX2 */
2290         c->hyscale_fast = NULL;
2291         c->hcscale_fast = NULL;
2292 #if COMPILE_TEMPLATE_MMX2
2293     }
2294 #endif /* COMPILE_TEMPLATE_MMX2 */
2295
2296 #if !COMPILE_TEMPLATE_MMX2
2297     switch(srcFormat) {
2298         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;
2299         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;
2300         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;
2301         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;
2302         case PIX_FMT_YUV420P16BE:
2303         case PIX_FMT_YUV422P16BE:
2304         case PIX_FMT_YUV444P16BE: c->chrToYV12 = RENAME(BEToUV); break;
2305         case PIX_FMT_YUV420P16LE:
2306         case PIX_FMT_YUV422P16LE:
2307         case PIX_FMT_YUV444P16LE: c->chrToYV12 = RENAME(LEToUV); break;
2308         default: break;
2309     }
2310 #endif /* !COMPILE_TEMPLATE_MMX2 */
2311     if (!c->chrSrcHSubSample) {
2312         switch(srcFormat) {
2313         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
2314         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
2315         default: break;
2316         }
2317     }
2318
2319     switch (srcFormat) {
2320 #if !COMPILE_TEMPLATE_MMX2
2321     case PIX_FMT_YUYV422  :
2322     case PIX_FMT_YUV420P16BE:
2323     case PIX_FMT_YUV422P16BE:
2324     case PIX_FMT_YUV444P16BE:
2325     case PIX_FMT_Y400A    :
2326     case PIX_FMT_GRAY16BE : c->lumToYV12 = RENAME(yuy2ToY); break;
2327     case PIX_FMT_UYVY422  :
2328     case PIX_FMT_YUV420P16LE:
2329     case PIX_FMT_YUV422P16LE:
2330     case PIX_FMT_YUV444P16LE:
2331     case PIX_FMT_GRAY16LE : c->lumToYV12 = RENAME(uyvyToY); break;
2332 #endif /* !COMPILE_TEMPLATE_MMX2 */
2333     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
2334     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
2335     default: break;
2336     }
2337 #if !COMPILE_TEMPLATE_MMX2
2338     if (c->alpPixBuf) {
2339         switch (srcFormat) {
2340         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;
2341         default: break;
2342         }
2343     }
2344 #endif /* !COMPILE_TEMPLATE_MMX2 */
2345 }