Compile x86/swscale_template with -mno-red-zone.
[ffmpeg.git] / libswscale / x86 / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #undef REAL_MOVNTQ
22 #undef MOVNTQ
23 #undef PREFETCH
24
25 #if COMPILE_TEMPLATE_MMX2
26 #define PREFETCH "prefetchnta"
27 #else
28 #define PREFETCH  " # nop"
29 #endif
30
31 #if COMPILE_TEMPLATE_MMX2
32 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
33 #else
34 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
35 #endif
36 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
37
38 #define YSCALEYUV2YV12X(offset, dest, end, pos) \
39     __asm__ volatile(\
40         "movq                  "DITHER16"+0(%0), %%mm3      \n\t"\
41         "movq                  "DITHER16"+8(%0), %%mm4      \n\t"\
42         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
43         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
44         ".p2align                             4             \n\t" /* FIXME Unroll? */\
45         "1:                                                 \n\t"\
46         "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\
47         "movq                (%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\
48         "movq               8(%%"REG_S", %3, 2), %%mm5      \n\t" /* srcData */\
49         "add                                $16, %%"REG_d"  \n\t"\
50         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
51         "test                         %%"REG_S", %%"REG_S"  \n\t"\
52         "pmulhw                           %%mm0, %%mm2      \n\t"\
53         "pmulhw                           %%mm0, %%mm5      \n\t"\
54         "paddw                            %%mm2, %%mm3      \n\t"\
55         "paddw                            %%mm5, %%mm4      \n\t"\
56         " jnz                                1b             \n\t"\
57         "psraw                               $3, %%mm3      \n\t"\
58         "psraw                               $3, %%mm4      \n\t"\
59         "packuswb                         %%mm4, %%mm3      \n\t"\
60         MOVNTQ(%%mm3, (%1, %3))\
61         "add                                 $8, %3         \n\t"\
62         "cmp                                 %2, %3         \n\t"\
63         "movq                  "DITHER16"+0(%0), %%mm3      \n\t"\
64         "movq                  "DITHER16"+8(%0), %%mm4      \n\t"\
65         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
66         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
67         "jb                                  1b             \n\t"\
68         :: "r" (&c->redDither),\
69            "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\
70         : "%"REG_d, "%"REG_S\
71     );
72
73 #if !COMPILE_TEMPLATE_MMX2
74 static av_always_inline void
75 dither_8to16(SwsContext *c, const uint8_t *srcDither, int rot)
76 {
77     if (rot) {
78         __asm__ volatile("pxor      %%mm0, %%mm0\n\t"
79                          "movq       (%0), %%mm3\n\t"
80                          "movq      %%mm3, %%mm4\n\t"
81                          "psrlq       $24, %%mm3\n\t"
82                          "psllq       $40, %%mm4\n\t"
83                          "por       %%mm4, %%mm3\n\t"
84                          "movq      %%mm3, %%mm4\n\t"
85                          "punpcklbw %%mm0, %%mm3\n\t"
86                          "punpckhbw %%mm0, %%mm4\n\t"
87                          "psraw        $4, %%mm3\n\t"
88                          "psraw        $4, %%mm4\n\t"
89                          "movq      %%mm3, "DITHER16"+0(%1)\n\t"
90                          "movq      %%mm4, "DITHER16"+8(%1)\n\t"
91                          :: "r"(srcDither), "r"(&c->redDither)
92                          );
93     } else {
94         __asm__ volatile("pxor      %%mm0, %%mm0\n\t"
95                          "movq       (%0), %%mm3\n\t"
96                          "movq      %%mm3, %%mm4\n\t"
97                          "punpcklbw %%mm0, %%mm3\n\t"
98                          "punpckhbw %%mm0, %%mm4\n\t"
99                          "psraw        $4, %%mm3\n\t"
100                          "psraw        $4, %%mm4\n\t"
101                          "movq      %%mm3, "DITHER16"+0(%1)\n\t"
102                          "movq      %%mm4, "DITHER16"+8(%1)\n\t"
103                          :: "r"(srcDither), "r"(&c->redDither)
104                          );
105     }
106 }
107 #endif
108
109 static void RENAME(yuv2yuvX)(SwsContext *c, const int16_t *lumFilter,
110                              const int16_t **lumSrc, int lumFilterSize,
111                              const int16_t *chrFilter, const int16_t **chrUSrc,
112                              const int16_t **chrVSrc,
113                              int chrFilterSize, const int16_t **alpSrc,
114                              uint8_t *dest[4], int dstW, int chrDstW)
115 {
116     uint8_t *yDest = dest[0], *uDest = dest[1], *vDest = dest[2],
117             *aDest = CONFIG_SWSCALE_ALPHA ? dest[3] : NULL;
118     const uint8_t *lumDither = c->lumDither8, *chrDither = c->chrDither8;
119
120     if (uDest) {
121         x86_reg uv_off = c->uv_offx2 >> 1;
122         dither_8to16(c, chrDither, 0);
123         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)
124         dither_8to16(c, chrDither, 1);
125         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, vDest - uv_off, chrDstW + uv_off, uv_off)
126     }
127     dither_8to16(c, lumDither, 0);
128     if (CONFIG_SWSCALE_ALPHA && aDest) {
129         YSCALEYUV2YV12X(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)
130     }
131
132     YSCALEYUV2YV12X(LUM_MMX_FILTER_OFFSET, yDest, dstW, 0)
133 }
134
135 #define YSCALEYUV2YV12X_ACCURATE(offset, dest, end, pos) \
136     __asm__ volatile(\
137         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
138         "movq                  "DITHER32"+0(%0), %%mm4      \n\t"\
139         "movq                  "DITHER32"+8(%0), %%mm5      \n\t"\
140         "movq                 "DITHER32"+16(%0), %%mm6      \n\t"\
141         "movq                 "DITHER32"+24(%0), %%mm7      \n\t"\
142         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
143         ".p2align                             4             \n\t"\
144         "1:                                                 \n\t"\
145         "movq                (%%"REG_S", %3, 2), %%mm0      \n\t" /* srcData */\
146         "movq               8(%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\
147         "mov        "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"  \n\t"\
148         "movq                (%%"REG_S", %3, 2), %%mm1      \n\t" /* srcData */\
149         "movq                             %%mm0, %%mm3      \n\t"\
150         "punpcklwd                        %%mm1, %%mm0      \n\t"\
151         "punpckhwd                        %%mm1, %%mm3      \n\t"\
152         "movq       "STR(APCK_COEF)"(%%"REG_d"), %%mm1      \n\t" /* filterCoeff */\
153         "pmaddwd                          %%mm1, %%mm0      \n\t"\
154         "pmaddwd                          %%mm1, %%mm3      \n\t"\
155         "paddd                            %%mm0, %%mm4      \n\t"\
156         "paddd                            %%mm3, %%mm5      \n\t"\
157         "movq               8(%%"REG_S", %3, 2), %%mm3      \n\t" /* srcData */\
158         "mov        "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"  \n\t"\
159         "add                  $"STR(APCK_SIZE)", %%"REG_d"  \n\t"\
160         "test                         %%"REG_S", %%"REG_S"  \n\t"\
161         "movq                             %%mm2, %%mm0      \n\t"\
162         "punpcklwd                        %%mm3, %%mm2      \n\t"\
163         "punpckhwd                        %%mm3, %%mm0      \n\t"\
164         "pmaddwd                          %%mm1, %%mm2      \n\t"\
165         "pmaddwd                          %%mm1, %%mm0      \n\t"\
166         "paddd                            %%mm2, %%mm6      \n\t"\
167         "paddd                            %%mm0, %%mm7      \n\t"\
168         " jnz                                1b             \n\t"\
169         "psrad                              $19, %%mm4      \n\t"\
170         "psrad                              $19, %%mm5      \n\t"\
171         "psrad                              $19, %%mm6      \n\t"\
172         "psrad                              $19, %%mm7      \n\t"\
173         "packssdw                         %%mm5, %%mm4      \n\t"\
174         "packssdw                         %%mm7, %%mm6      \n\t"\
175         "packuswb                         %%mm6, %%mm4      \n\t"\
176         MOVNTQ(%%mm4, (%1, %3))\
177         "add                                 $8, %3         \n\t"\
178         "cmp                                 %2, %3         \n\t"\
179         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
180         "movq                  "DITHER32"+0(%0), %%mm4      \n\t"\
181         "movq                  "DITHER32"+8(%0), %%mm5      \n\t"\
182         "movq                 "DITHER32"+16(%0), %%mm6      \n\t"\
183         "movq                 "DITHER32"+24(%0), %%mm7      \n\t"\
184         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
185         "jb                                  1b             \n\t"\
186         :: "r" (&c->redDither),\
187         "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\
188         : "%"REG_a, "%"REG_d, "%"REG_S\
189     );
190
191 #if !COMPILE_TEMPLATE_MMX2
192 static av_always_inline void
193 dither_8to32(SwsContext *c, const uint8_t *srcDither, int rot)
194 {
195     if (rot) {
196         __asm__ volatile("pxor      %%mm0, %%mm0\n\t"
197                          "movq       (%0), %%mm4\n\t"
198                          "movq      %%mm4, %%mm5\n\t"
199                          "psrlq       $24, %%mm4\n\t"
200                          "psllq       $40, %%mm5\n\t"
201                          "por       %%mm5, %%mm4\n\t"
202                          "movq      %%mm4, %%mm6\n\t"
203                          "punpcklbw %%mm0, %%mm4\n\t"
204                          "punpckhbw %%mm0, %%mm6\n\t"
205                          "movq      %%mm4, %%mm5\n\t"
206                          "movq      %%mm6, %%mm7\n\t"
207                          "punpcklwd %%mm0, %%mm4\n\t"
208                          "punpckhwd %%mm0, %%mm5\n\t"
209                          "punpcklwd %%mm0, %%mm6\n\t"
210                          "punpckhwd %%mm0, %%mm7\n\t"
211                          "pslld       $12, %%mm4\n\t"
212                          "pslld       $12, %%mm5\n\t"
213                          "pslld       $12, %%mm6\n\t"
214                          "pslld       $12, %%mm7\n\t"
215                          "movq      %%mm4, "DITHER32"+0(%1)\n\t"
216                          "movq      %%mm5, "DITHER32"+8(%1)\n\t"
217                          "movq      %%mm6, "DITHER32"+16(%1)\n\t"
218                          "movq      %%mm7, "DITHER32"+24(%1)\n\t"
219                          :: "r"(srcDither), "r"(&c->redDither)
220                          );
221     } else {
222         __asm__ volatile("pxor      %%mm0, %%mm0\n\t"
223                          "movq       (%0), %%mm4\n\t"
224                          "movq      %%mm4, %%mm6\n\t"
225                          "punpcklbw %%mm0, %%mm4\n\t"
226                          "punpckhbw %%mm0, %%mm6\n\t"
227                          "movq      %%mm4, %%mm5\n\t"
228                          "movq      %%mm6, %%mm7\n\t"
229                          "punpcklwd %%mm0, %%mm4\n\t"
230                          "punpckhwd %%mm0, %%mm5\n\t"
231                          "punpcklwd %%mm0, %%mm6\n\t"
232                          "punpckhwd %%mm0, %%mm7\n\t"
233                          "pslld       $12, %%mm4\n\t"
234                          "pslld       $12, %%mm5\n\t"
235                          "pslld       $12, %%mm6\n\t"
236                          "pslld       $12, %%mm7\n\t"
237                          "movq      %%mm4, "DITHER32"+0(%1)\n\t"
238                          "movq      %%mm5, "DITHER32"+8(%1)\n\t"
239                          "movq      %%mm6, "DITHER32"+16(%1)\n\t"
240                          "movq      %%mm7, "DITHER32"+24(%1)\n\t"
241                          :: "r"(srcDither), "r"(&c->redDither)
242                          );
243     }
244 }
245 #endif
246
247 static void RENAME(yuv2yuvX_ar)(SwsContext *c, const int16_t *lumFilter,
248                                 const int16_t **lumSrc, int lumFilterSize,
249                                 const int16_t *chrFilter, const int16_t **chrUSrc,
250                                 const int16_t **chrVSrc,
251                                 int chrFilterSize, const int16_t **alpSrc,
252                                 uint8_t *dest[4], int dstW, int chrDstW)
253 {
254     uint8_t *yDest = dest[0], *uDest = dest[1], *vDest = dest[2],
255             *aDest = CONFIG_SWSCALE_ALPHA ? dest[3] : NULL;
256     const uint8_t *lumDither = c->lumDither8, *chrDither = c->chrDither8;
257
258     if (uDest) {
259         x86_reg uv_off = c->uv_offx2 >> 1;
260         dither_8to32(c, chrDither, 0);
261         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)
262         dither_8to32(c, chrDither, 1);
263         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, vDest - uv_off, chrDstW + uv_off, uv_off)
264     }
265     dither_8to32(c, lumDither, 0);
266     if (CONFIG_SWSCALE_ALPHA && aDest) {
267         YSCALEYUV2YV12X_ACCURATE(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)
268     }
269
270     YSCALEYUV2YV12X_ACCURATE(LUM_MMX_FILTER_OFFSET, yDest, dstW, 0)
271 }
272
273 static void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc,
274                              const int16_t *chrUSrc, const int16_t *chrVSrc,
275                              const int16_t *alpSrc,
276                              uint8_t *dst[4], int dstW, int chrDstW)
277 {
278     int p= 4;
279     const int16_t *src[4]= {
280         lumSrc + dstW,     chrUSrc + chrDstW,
281         chrVSrc + chrDstW, alpSrc + dstW
282     };
283     x86_reg counter[4]= { dstW, chrDstW, chrDstW, dstW };
284
285     while (p--) {
286         if (dst[p]) {
287             __asm__ volatile(
288                 "mov %2, %%"REG_a"                    \n\t"
289                 ".p2align               4             \n\t" /* FIXME Unroll? */
290                 "1:                                   \n\t"
291                 "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"
292                 "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"
293                 "psraw                 $7, %%mm0      \n\t"
294                 "psraw                 $7, %%mm1      \n\t"
295                 "packuswb           %%mm1, %%mm0      \n\t"
296                 MOVNTQ(%%mm0, (%1, %%REGa))
297                 "add                   $8, %%"REG_a"  \n\t"
298                 "jnc                   1b             \n\t"
299                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
300                    "g" (-counter[p])
301                 : "%"REG_a
302             );
303         }
304     }
305 }
306
307 static void RENAME(yuv2yuv1_ar)(SwsContext *c, const int16_t *lumSrc,
308                                 const int16_t *chrUSrc, const int16_t *chrVSrc,
309                                 const int16_t *alpSrc,
310                                 uint8_t *dst[4], int dstW, int chrDstW)
311 {
312     int p= 4;
313     const int16_t *src[4]= {
314         lumSrc + dstW,     chrUSrc + chrDstW,
315         chrVSrc + chrDstW, alpSrc + dstW
316     };
317     x86_reg counter[4]= { dstW, chrDstW, chrDstW, dstW };
318     const uint8_t *lumDither = c->lumDither8, *chrDither = c->chrDither8;
319
320     while (p--) {
321         if (dst[p]) {
322             int i;
323             for(i=0; i<8; i++) c->dither16[i] = (p == 2 || p == 3) ? lumDither[i] : chrDither[i];
324             __asm__ volatile(
325                 "mov %2, %%"REG_a"                    \n\t"
326                 "movq    "DITHER16"+0(%3), %%mm6      \n\t"
327                 "movq    "DITHER16"+8(%3), %%mm7      \n\t"
328                 ".p2align                4            \n\t" /* FIXME Unroll? */
329                 "1:                                   \n\t"
330                 "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"
331                 "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"
332                 "paddsw             %%mm6, %%mm0      \n\t"
333                 "paddsw             %%mm7, %%mm1      \n\t"
334                 "psraw                 $7, %%mm0      \n\t"
335                 "psraw                 $7, %%mm1      \n\t"
336                 "packuswb           %%mm1, %%mm0      \n\t"
337                 MOVNTQ(%%mm0, (%1, %%REGa))
338                 "add                   $8, %%"REG_a"  \n\t"
339                 "jnc                   1b             \n\t"
340                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
341                    "g" (-counter[p]), "r"(&c->redDither)
342                 : "%"REG_a
343             );
344         }
345     }
346 }
347
348 #define YSCALEYUV2PACKEDX_UV \
349     __asm__ volatile(\
350         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
351         ".p2align                      4                \n\t"\
352         "nop                                            \n\t"\
353         "1:                                             \n\t"\
354         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
355         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
356         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
357         "movq                      %%mm3, %%mm4         \n\t"\
358         ".p2align                      4                \n\t"\
359         "2:                                             \n\t"\
360         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
361         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
362         "add                          %6, %%"REG_S"     \n\t" \
363         "movq     (%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
364         "add                         $16, %%"REG_d"     \n\t"\
365         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
366         "pmulhw                    %%mm0, %%mm2         \n\t"\
367         "pmulhw                    %%mm0, %%mm5         \n\t"\
368         "paddw                     %%mm2, %%mm3         \n\t"\
369         "paddw                     %%mm5, %%mm4         \n\t"\
370         "test                  %%"REG_S", %%"REG_S"     \n\t"\
371         " jnz                         2b                \n\t"\
372
373 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
374     "lea                "offset"(%0), %%"REG_d"     \n\t"\
375     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
376     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
377     "movq                    "#dst1", "#dst2"       \n\t"\
378     ".p2align                      4                \n\t"\
379     "2:                                             \n\t"\
380     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
381     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
382     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
383     "add                         $16, %%"REG_d"            \n\t"\
384     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
385     "pmulhw                 "#coeff", "#src1"       \n\t"\
386     "pmulhw                 "#coeff", "#src2"       \n\t"\
387     "paddw                   "#src1", "#dst1"       \n\t"\
388     "paddw                   "#src2", "#dst2"       \n\t"\
389     "test                  %%"REG_S", %%"REG_S"     \n\t"\
390     " jnz                         2b                \n\t"\
391
392 #define YSCALEYUV2PACKEDX \
393     YSCALEYUV2PACKEDX_UV \
394     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
395
396 #define YSCALEYUV2PACKEDX_END                     \
397         :: "r" (&c->redDither),                   \
398             "m" (dummy), "m" (dummy), "m" (dummy),\
399             "r" (dest), "m" (dstW_reg), "m"(uv_off) \
400         : "%"REG_a, "%"REG_d, "%"REG_S            \
401     );
402
403 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
404     __asm__ volatile(\
405         "xor %%"REG_a", %%"REG_a"                       \n\t"\
406         ".p2align                      4                \n\t"\
407         "nop                                            \n\t"\
408         "1:                                             \n\t"\
409         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
410         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
411         "pxor                      %%mm4, %%mm4         \n\t"\
412         "pxor                      %%mm5, %%mm5         \n\t"\
413         "pxor                      %%mm6, %%mm6         \n\t"\
414         "pxor                      %%mm7, %%mm7         \n\t"\
415         ".p2align                      4                \n\t"\
416         "2:                                             \n\t"\
417         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
418         "add                          %6, %%"REG_S"      \n\t" \
419         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
420         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
421         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
422         "movq                      %%mm0, %%mm3         \n\t"\
423         "punpcklwd                 %%mm1, %%mm0         \n\t"\
424         "punpckhwd                 %%mm1, %%mm3         \n\t"\
425         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
426         "pmaddwd                   %%mm1, %%mm0         \n\t"\
427         "pmaddwd                   %%mm1, %%mm3         \n\t"\
428         "paddd                     %%mm0, %%mm4         \n\t"\
429         "paddd                     %%mm3, %%mm5         \n\t"\
430         "add                          %6, %%"REG_S"      \n\t" \
431         "movq     (%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
432         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
433         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
434         "test                  %%"REG_S", %%"REG_S"     \n\t"\
435         "movq                      %%mm2, %%mm0         \n\t"\
436         "punpcklwd                 %%mm3, %%mm2         \n\t"\
437         "punpckhwd                 %%mm3, %%mm0         \n\t"\
438         "pmaddwd                   %%mm1, %%mm2         \n\t"\
439         "pmaddwd                   %%mm1, %%mm0         \n\t"\
440         "paddd                     %%mm2, %%mm6         \n\t"\
441         "paddd                     %%mm0, %%mm7         \n\t"\
442         " jnz                         2b                \n\t"\
443         "psrad                       $16, %%mm4         \n\t"\
444         "psrad                       $16, %%mm5         \n\t"\
445         "psrad                       $16, %%mm6         \n\t"\
446         "psrad                       $16, %%mm7         \n\t"\
447         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
448         "packssdw                  %%mm5, %%mm4         \n\t"\
449         "packssdw                  %%mm7, %%mm6         \n\t"\
450         "paddw                     %%mm0, %%mm4         \n\t"\
451         "paddw                     %%mm0, %%mm6         \n\t"\
452         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
453         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
454
455 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
456     "lea                "offset"(%0), %%"REG_d"     \n\t"\
457     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
458     "pxor                      %%mm1, %%mm1         \n\t"\
459     "pxor                      %%mm5, %%mm5         \n\t"\
460     "pxor                      %%mm7, %%mm7         \n\t"\
461     "pxor                      %%mm6, %%mm6         \n\t"\
462     ".p2align                      4                \n\t"\
463     "2:                                             \n\t"\
464     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
465     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
466     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
467     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
468     "movq                      %%mm0, %%mm3         \n\t"\
469     "punpcklwd                 %%mm4, %%mm0         \n\t"\
470     "punpckhwd                 %%mm4, %%mm3         \n\t"\
471     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
472     "pmaddwd                   %%mm4, %%mm0         \n\t"\
473     "pmaddwd                   %%mm4, %%mm3         \n\t"\
474     "paddd                     %%mm0, %%mm1         \n\t"\
475     "paddd                     %%mm3, %%mm5         \n\t"\
476     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
477     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
478     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
479     "test                  %%"REG_S", %%"REG_S"     \n\t"\
480     "movq                      %%mm2, %%mm0         \n\t"\
481     "punpcklwd                 %%mm3, %%mm2         \n\t"\
482     "punpckhwd                 %%mm3, %%mm0         \n\t"\
483     "pmaddwd                   %%mm4, %%mm2         \n\t"\
484     "pmaddwd                   %%mm4, %%mm0         \n\t"\
485     "paddd                     %%mm2, %%mm7         \n\t"\
486     "paddd                     %%mm0, %%mm6         \n\t"\
487     " jnz                         2b                \n\t"\
488     "psrad                       $16, %%mm1         \n\t"\
489     "psrad                       $16, %%mm5         \n\t"\
490     "psrad                       $16, %%mm7         \n\t"\
491     "psrad                       $16, %%mm6         \n\t"\
492     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
493     "packssdw                  %%mm5, %%mm1         \n\t"\
494     "packssdw                  %%mm6, %%mm7         \n\t"\
495     "paddw                     %%mm0, %%mm1         \n\t"\
496     "paddw                     %%mm0, %%mm7         \n\t"\
497     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
498     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
499
500 #define YSCALEYUV2PACKEDX_ACCURATE \
501     YSCALEYUV2PACKEDX_ACCURATE_UV \
502     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
503
504 #define YSCALEYUV2RGBX \
505     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
506     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
507     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
508     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
509     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
510     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
511     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
512     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
513     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
514     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
515     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
516     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
517     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
518     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
519     "paddw           %%mm3, %%mm4       \n\t"\
520     "movq            %%mm2, %%mm0       \n\t"\
521     "movq            %%mm5, %%mm6       \n\t"\
522     "movq            %%mm4, %%mm3       \n\t"\
523     "punpcklwd       %%mm2, %%mm2       \n\t"\
524     "punpcklwd       %%mm5, %%mm5       \n\t"\
525     "punpcklwd       %%mm4, %%mm4       \n\t"\
526     "paddw           %%mm1, %%mm2       \n\t"\
527     "paddw           %%mm1, %%mm5       \n\t"\
528     "paddw           %%mm1, %%mm4       \n\t"\
529     "punpckhwd       %%mm0, %%mm0       \n\t"\
530     "punpckhwd       %%mm6, %%mm6       \n\t"\
531     "punpckhwd       %%mm3, %%mm3       \n\t"\
532     "paddw           %%mm7, %%mm0       \n\t"\
533     "paddw           %%mm7, %%mm6       \n\t"\
534     "paddw           %%mm7, %%mm3       \n\t"\
535     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
536     "packuswb        %%mm0, %%mm2       \n\t"\
537     "packuswb        %%mm6, %%mm5       \n\t"\
538     "packuswb        %%mm3, %%mm4       \n\t"\
539
540 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
541     "movq       "#b", "#q2"     \n\t" /* B */\
542     "movq       "#r", "#t"      \n\t" /* R */\
543     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
544     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
545     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
546     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
547     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
548     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
549     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
550     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
551     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
552     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
553 \
554     MOVNTQ(   q0,   (dst, index, 4))\
555     MOVNTQ(    b,  8(dst, index, 4))\
556     MOVNTQ(   q2, 16(dst, index, 4))\
557     MOVNTQ(   q3, 24(dst, index, 4))\
558 \
559     "add      $8, "#index"      \n\t"\
560     "cmp "#dstw", "#index"      \n\t"\
561     " jb      1b                \n\t"
562 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
563
564 static void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,
565                                    const int16_t **lumSrc, int lumFilterSize,
566                                    const int16_t *chrFilter, const int16_t **chrUSrc,
567                                    const int16_t **chrVSrc,
568                                    int chrFilterSize, const int16_t **alpSrc,
569                                    uint8_t *dest, int dstW, int dstY)
570 {
571     x86_reg dummy=0;
572     x86_reg dstW_reg = dstW;
573     x86_reg uv_off = c->uv_offx2;
574
575     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
576         YSCALEYUV2PACKEDX_ACCURATE
577         YSCALEYUV2RGBX
578         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
579         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
580         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
581         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
582         "movq               "Y_TEMP"(%0), %%mm5         \n\t"
583         "psraw                        $3, %%mm1         \n\t"
584         "psraw                        $3, %%mm7         \n\t"
585         "packuswb                  %%mm7, %%mm1         \n\t"
586         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
587         YSCALEYUV2PACKEDX_END
588     } else {
589         YSCALEYUV2PACKEDX_ACCURATE
590         YSCALEYUV2RGBX
591         "pcmpeqd %%mm7, %%mm7 \n\t"
592         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
593         YSCALEYUV2PACKEDX_END
594     }
595 }
596
597 static void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,
598                                 const int16_t **lumSrc, int lumFilterSize,
599                                 const int16_t *chrFilter, const int16_t **chrUSrc,
600                                 const int16_t **chrVSrc,
601                                 int chrFilterSize, const int16_t **alpSrc,
602                                 uint8_t *dest, int dstW, int dstY)
603 {
604     x86_reg dummy=0;
605     x86_reg dstW_reg = dstW;
606     x86_reg uv_off = c->uv_offx2;
607
608     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
609         YSCALEYUV2PACKEDX
610         YSCALEYUV2RGBX
611         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
612         "psraw                        $3, %%mm1         \n\t"
613         "psraw                        $3, %%mm7         \n\t"
614         "packuswb                  %%mm7, %%mm1         \n\t"
615         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
616         YSCALEYUV2PACKEDX_END
617     } else {
618         YSCALEYUV2PACKEDX
619         YSCALEYUV2RGBX
620         "pcmpeqd %%mm7, %%mm7 \n\t"
621         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
622         YSCALEYUV2PACKEDX_END
623     }
624 }
625
626 #define REAL_WRITERGB16(dst, dstw, index) \
627     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
628     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
629     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
630     "psrlq           $3, %%mm2  \n\t"\
631 \
632     "movq         %%mm2, %%mm1  \n\t"\
633     "movq         %%mm4, %%mm3  \n\t"\
634 \
635     "punpcklbw    %%mm7, %%mm3  \n\t"\
636     "punpcklbw    %%mm5, %%mm2  \n\t"\
637     "punpckhbw    %%mm7, %%mm4  \n\t"\
638     "punpckhbw    %%mm5, %%mm1  \n\t"\
639 \
640     "psllq           $3, %%mm3  \n\t"\
641     "psllq           $3, %%mm4  \n\t"\
642 \
643     "por          %%mm3, %%mm2  \n\t"\
644     "por          %%mm4, %%mm1  \n\t"\
645 \
646     MOVNTQ(%%mm2,  (dst, index, 2))\
647     MOVNTQ(%%mm1, 8(dst, index, 2))\
648 \
649     "add             $8, "#index"   \n\t"\
650     "cmp        "#dstw", "#index"   \n\t"\
651     " jb             1b             \n\t"
652 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
653
654 static void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,
655                                     const int16_t **lumSrc, int lumFilterSize,
656                                     const int16_t *chrFilter, const int16_t **chrUSrc,
657                                     const int16_t **chrVSrc,
658                                     int chrFilterSize, const int16_t **alpSrc,
659                                     uint8_t *dest, int dstW, int dstY)
660 {
661     x86_reg dummy=0;
662     x86_reg dstW_reg = dstW;
663     x86_reg uv_off = c->uv_offx2;
664
665     YSCALEYUV2PACKEDX_ACCURATE
666     YSCALEYUV2RGBX
667     "pxor %%mm7, %%mm7 \n\t"
668     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
669 #ifdef DITHER1XBPP
670     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
671     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
672     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
673 #endif
674     WRITERGB16(%4, %5, %%REGa)
675     YSCALEYUV2PACKEDX_END
676 }
677
678 static void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,
679                                  const int16_t **lumSrc, int lumFilterSize,
680                                  const int16_t *chrFilter, const int16_t **chrUSrc,
681                                  const int16_t **chrVSrc,
682                                  int chrFilterSize, const int16_t **alpSrc,
683                                  uint8_t *dest, int dstW, int dstY)
684 {
685     x86_reg dummy=0;
686     x86_reg dstW_reg = dstW;
687     x86_reg uv_off = c->uv_offx2;
688
689     YSCALEYUV2PACKEDX
690     YSCALEYUV2RGBX
691     "pxor %%mm7, %%mm7 \n\t"
692     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
693 #ifdef DITHER1XBPP
694     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
695     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
696     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
697 #endif
698     WRITERGB16(%4, %5, %%REGa)
699     YSCALEYUV2PACKEDX_END
700 }
701
702 #define REAL_WRITERGB15(dst, dstw, index) \
703     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
704     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
705     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
706     "psrlq           $3, %%mm2  \n\t"\
707     "psrlq           $1, %%mm5  \n\t"\
708 \
709     "movq         %%mm2, %%mm1  \n\t"\
710     "movq         %%mm4, %%mm3  \n\t"\
711 \
712     "punpcklbw    %%mm7, %%mm3  \n\t"\
713     "punpcklbw    %%mm5, %%mm2  \n\t"\
714     "punpckhbw    %%mm7, %%mm4  \n\t"\
715     "punpckhbw    %%mm5, %%mm1  \n\t"\
716 \
717     "psllq           $2, %%mm3  \n\t"\
718     "psllq           $2, %%mm4  \n\t"\
719 \
720     "por          %%mm3, %%mm2  \n\t"\
721     "por          %%mm4, %%mm1  \n\t"\
722 \
723     MOVNTQ(%%mm2,  (dst, index, 2))\
724     MOVNTQ(%%mm1, 8(dst, index, 2))\
725 \
726     "add             $8, "#index"   \n\t"\
727     "cmp        "#dstw", "#index"   \n\t"\
728     " jb             1b             \n\t"
729 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
730
731 static void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,
732                                     const int16_t **lumSrc, int lumFilterSize,
733                                     const int16_t *chrFilter, const int16_t **chrUSrc,
734                                     const int16_t **chrVSrc,
735                                     int chrFilterSize, const int16_t **alpSrc,
736                                     uint8_t *dest, int dstW, int dstY)
737 {
738     x86_reg dummy=0;
739     x86_reg dstW_reg = dstW;
740     x86_reg uv_off = c->uv_offx2;
741
742     YSCALEYUV2PACKEDX_ACCURATE
743     YSCALEYUV2RGBX
744     "pxor %%mm7, %%mm7 \n\t"
745     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
746 #ifdef DITHER1XBPP
747     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
748     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
749     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
750 #endif
751     WRITERGB15(%4, %5, %%REGa)
752     YSCALEYUV2PACKEDX_END
753 }
754
755 static void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,
756                                  const int16_t **lumSrc, int lumFilterSize,
757                                  const int16_t *chrFilter, const int16_t **chrUSrc,
758                                  const int16_t **chrVSrc,
759                                  int chrFilterSize, const int16_t **alpSrc,
760                                  uint8_t *dest, int dstW, int dstY)
761 {
762     x86_reg dummy=0;
763     x86_reg dstW_reg = dstW;
764     x86_reg uv_off = c->uv_offx2;
765
766     YSCALEYUV2PACKEDX
767     YSCALEYUV2RGBX
768     "pxor %%mm7, %%mm7 \n\t"
769     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
770 #ifdef DITHER1XBPP
771     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
772     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
773     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
774 #endif
775     WRITERGB15(%4, %5, %%REGa)
776     YSCALEYUV2PACKEDX_END
777 }
778
779 #define WRITEBGR24MMX(dst, dstw, index) \
780     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
781     "movq      %%mm2, %%mm1     \n\t" /* B */\
782     "movq      %%mm5, %%mm6     \n\t" /* R */\
783     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
784     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
785     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
786     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
787     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
788     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
789     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
790     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
791     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
792     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
793 \
794     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
795     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
796     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
797     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
798 \
799     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
800     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
801     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
802     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
803 \
804     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
805     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
806     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
807     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
808 \
809     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
810     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
811     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
812     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
813     MOVNTQ(%%mm0, (dst))\
814 \
815     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
816     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
817     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
818     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
819     MOVNTQ(%%mm6, 8(dst))\
820 \
821     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
822     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
823     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
824     MOVNTQ(%%mm5, 16(dst))\
825 \
826     "add         $24, "#dst"    \n\t"\
827 \
828     "add          $8, "#index"  \n\t"\
829     "cmp     "#dstw", "#index"  \n\t"\
830     " jb          1b            \n\t"
831
832 #define WRITEBGR24MMX2(dst, dstw, index) \
833     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
834     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
835     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
836     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
837     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
838     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
839 \
840     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
841     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
842     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
843 \
844     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
845     "por    %%mm1, %%mm6        \n\t"\
846     "por    %%mm3, %%mm6        \n\t"\
847     MOVNTQ(%%mm6, (dst))\
848 \
849     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
850     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
851     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
852     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
853 \
854     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
855     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
856     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
857 \
858     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
859     "por    %%mm3, %%mm6        \n\t"\
860     MOVNTQ(%%mm6, 8(dst))\
861 \
862     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
863     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
864     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
865 \
866     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
867     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
868     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
869 \
870     "por    %%mm1, %%mm3        \n\t"\
871     "por    %%mm3, %%mm6        \n\t"\
872     MOVNTQ(%%mm6, 16(dst))\
873 \
874     "add      $24, "#dst"       \n\t"\
875 \
876     "add       $8, "#index"     \n\t"\
877     "cmp  "#dstw", "#index"     \n\t"\
878     " jb       1b               \n\t"
879
880 #if COMPILE_TEMPLATE_MMX2
881 #undef WRITEBGR24
882 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
883 #else
884 #undef WRITEBGR24
885 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
886 #endif
887
888 static void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,
889                                    const int16_t **lumSrc, int lumFilterSize,
890                                    const int16_t *chrFilter, const int16_t **chrUSrc,
891                                    const int16_t **chrVSrc,
892                                    int chrFilterSize, const int16_t **alpSrc,
893                                    uint8_t *dest, int dstW, int dstY)
894 {
895     x86_reg dummy=0;
896     x86_reg dstW_reg = dstW;
897     x86_reg uv_off = c->uv_offx2;
898
899     YSCALEYUV2PACKEDX_ACCURATE
900     YSCALEYUV2RGBX
901     "pxor %%mm7, %%mm7 \n\t"
902     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
903     "add %4, %%"REG_c"                        \n\t"
904     WRITEBGR24(%%REGc, %5, %%REGa)
905     :: "r" (&c->redDither),
906        "m" (dummy), "m" (dummy), "m" (dummy),
907        "r" (dest), "m" (dstW_reg), "m"(uv_off)
908     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
909     );
910 }
911
912 static void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,
913                                 const int16_t **lumSrc, int lumFilterSize,
914                                 const int16_t *chrFilter, const int16_t **chrUSrc,
915                                 const int16_t **chrVSrc,
916                                 int chrFilterSize, const int16_t **alpSrc,
917                                 uint8_t *dest, int dstW, int dstY)
918 {
919     x86_reg dummy=0;
920     x86_reg dstW_reg = dstW;
921     x86_reg uv_off = c->uv_offx2;
922
923     YSCALEYUV2PACKEDX
924     YSCALEYUV2RGBX
925     "pxor                    %%mm7, %%mm7       \n\t"
926     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
927     "add                        %4, %%"REG_c"   \n\t"
928     WRITEBGR24(%%REGc, %5, %%REGa)
929     :: "r" (&c->redDither),
930        "m" (dummy), "m" (dummy), "m" (dummy),
931        "r" (dest),  "m" (dstW_reg), "m"(uv_off)
932     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
933     );
934 }
935
936 #define REAL_WRITEYUY2(dst, dstw, index) \
937     "packuswb  %%mm3, %%mm3     \n\t"\
938     "packuswb  %%mm4, %%mm4     \n\t"\
939     "packuswb  %%mm7, %%mm1     \n\t"\
940     "punpcklbw %%mm4, %%mm3     \n\t"\
941     "movq      %%mm1, %%mm7     \n\t"\
942     "punpcklbw %%mm3, %%mm1     \n\t"\
943     "punpckhbw %%mm3, %%mm7     \n\t"\
944 \
945     MOVNTQ(%%mm1, (dst, index, 2))\
946     MOVNTQ(%%mm7, 8(dst, index, 2))\
947 \
948     "add          $8, "#index"  \n\t"\
949     "cmp     "#dstw", "#index"  \n\t"\
950     " jb          1b            \n\t"
951 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
952
953 static void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,
954                                      const int16_t **lumSrc, int lumFilterSize,
955                                      const int16_t *chrFilter, const int16_t **chrUSrc,
956                                      const int16_t **chrVSrc,
957                                      int chrFilterSize, const int16_t **alpSrc,
958                                      uint8_t *dest, int dstW, int dstY)
959 {
960     x86_reg dummy=0;
961     x86_reg dstW_reg = dstW;
962     x86_reg uv_off = c->uv_offx2;
963
964     YSCALEYUV2PACKEDX_ACCURATE
965     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
966     "psraw $3, %%mm3    \n\t"
967     "psraw $3, %%mm4    \n\t"
968     "psraw $3, %%mm1    \n\t"
969     "psraw $3, %%mm7    \n\t"
970     WRITEYUY2(%4, %5, %%REGa)
971     YSCALEYUV2PACKEDX_END
972 }
973
974 static void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,
975                                   const int16_t **lumSrc, int lumFilterSize,
976                                   const int16_t *chrFilter, const int16_t **chrUSrc,
977                                   const int16_t **chrVSrc,
978                                   int chrFilterSize, const int16_t **alpSrc,
979                                   uint8_t *dest, int dstW, int dstY)
980 {
981     x86_reg dummy=0;
982     x86_reg dstW_reg = dstW;
983     x86_reg uv_off = c->uv_offx2;
984
985     YSCALEYUV2PACKEDX
986     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
987     "psraw $3, %%mm3    \n\t"
988     "psraw $3, %%mm4    \n\t"
989     "psraw $3, %%mm1    \n\t"
990     "psraw $3, %%mm7    \n\t"
991     WRITEYUY2(%4, %5, %%REGa)
992     YSCALEYUV2PACKEDX_END
993 }
994
995 #define REAL_YSCALEYUV2RGB_UV(index, c) \
996     "xor            "#index", "#index"  \n\t"\
997     ".p2align              4            \n\t"\
998     "1:                                 \n\t"\
999     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1000     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1001     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1002     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1003     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1004     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1005     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
1006     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
1007     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
1008     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
1009     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
1010     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1011     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1012     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
1013     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
1014     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1015     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1016     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1017     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1018     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1019     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1020     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1021
1022 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
1023     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
1024     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
1025     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
1026     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
1027     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
1028     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
1029     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1030     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1031     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1032     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1033     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1034     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1035
1036 #define REAL_YSCALEYUV2RGB_COEFF(c) \
1037     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1038     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1039     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1040     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1041     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1042     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1043     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1044     "paddw             %%mm3, %%mm4     \n\t"\
1045     "movq              %%mm2, %%mm0     \n\t"\
1046     "movq              %%mm5, %%mm6     \n\t"\
1047     "movq              %%mm4, %%mm3     \n\t"\
1048     "punpcklwd         %%mm2, %%mm2     \n\t"\
1049     "punpcklwd         %%mm5, %%mm5     \n\t"\
1050     "punpcklwd         %%mm4, %%mm4     \n\t"\
1051     "paddw             %%mm1, %%mm2     \n\t"\
1052     "paddw             %%mm1, %%mm5     \n\t"\
1053     "paddw             %%mm1, %%mm4     \n\t"\
1054     "punpckhwd         %%mm0, %%mm0     \n\t"\
1055     "punpckhwd         %%mm6, %%mm6     \n\t"\
1056     "punpckhwd         %%mm3, %%mm3     \n\t"\
1057     "paddw             %%mm7, %%mm0     \n\t"\
1058     "paddw             %%mm7, %%mm6     \n\t"\
1059     "paddw             %%mm7, %%mm3     \n\t"\
1060     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1061     "packuswb          %%mm0, %%mm2     \n\t"\
1062     "packuswb          %%mm6, %%mm5     \n\t"\
1063     "packuswb          %%mm3, %%mm4     \n\t"\
1064
1065 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
1066
1067 #define YSCALEYUV2RGB(index, c) \
1068     REAL_YSCALEYUV2RGB_UV(index, c) \
1069     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
1070     REAL_YSCALEYUV2RGB_COEFF(c)
1071
1072 /**
1073  * vertical bilinear scale YV12 to RGB
1074  */
1075 static void RENAME(yuv2rgb32_2)(SwsContext *c, const int16_t *buf[2],
1076                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
1077                                 const int16_t *abuf[2], uint8_t *dest,
1078                                 int dstW, int yalpha, int uvalpha, int y)
1079 {
1080     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1081                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1082
1083     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1084         const int16_t *abuf0 = abuf[0], *abuf1 = abuf[1];
1085 #if ARCH_X86_64
1086         __asm__ volatile(
1087             YSCALEYUV2RGB(%%r8, %5)
1088             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
1089             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1090             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1091             "packuswb            %%mm7, %%mm1       \n\t"
1092             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1093             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "r" (dest),
1094                "a" (&c->redDither),
1095                "r" (abuf0), "r" (abuf1)
1096             : "%r8"
1097         );
1098 #else
1099         c->u_temp=(intptr_t)abuf0;
1100         c->v_temp=(intptr_t)abuf1;
1101         __asm__ volatile(
1102             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1103             "mov        %4, %%"REG_b"               \n\t"
1104             "push %%"REG_BP"                        \n\t"
1105             YSCALEYUV2RGB(%%REGBP, %5)
1106             "push                   %0              \n\t"
1107             "push                   %1              \n\t"
1108             "mov          "U_TEMP"(%5), %0          \n\t"
1109             "mov          "V_TEMP"(%5), %1          \n\t"
1110             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
1111             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1112             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1113             "packuswb            %%mm7, %%mm1       \n\t"
1114             "pop                    %1              \n\t"
1115             "pop                    %0              \n\t"
1116             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1117             "pop %%"REG_BP"                         \n\t"
1118             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1119             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1120                "a" (&c->redDither)
1121         );
1122 #endif
1123     } else {
1124         __asm__ volatile(
1125             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1126             "mov        %4, %%"REG_b"               \n\t"
1127             "push %%"REG_BP"                        \n\t"
1128             YSCALEYUV2RGB(%%REGBP, %5)
1129             "pcmpeqd %%mm7, %%mm7                   \n\t"
1130             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1131             "pop %%"REG_BP"                         \n\t"
1132             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1133             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1134                "a" (&c->redDither)
1135         );
1136     }
1137 }
1138
1139 static void RENAME(yuv2bgr24_2)(SwsContext *c, const int16_t *buf[2],
1140                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
1141                                 const int16_t *abuf[2], uint8_t *dest,
1142                                 int dstW, int yalpha, int uvalpha, int y)
1143 {
1144     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1145                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1146
1147     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1148     __asm__ volatile(
1149         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1150         "mov        %4, %%"REG_b"               \n\t"
1151         "push %%"REG_BP"                        \n\t"
1152         YSCALEYUV2RGB(%%REGBP, %5)
1153         "pxor    %%mm7, %%mm7                   \n\t"
1154         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1155         "pop %%"REG_BP"                         \n\t"
1156         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1157         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1158            "a" (&c->redDither)
1159     );
1160 }
1161
1162 static void RENAME(yuv2rgb555_2)(SwsContext *c, const int16_t *buf[2],
1163                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
1164                                  const int16_t *abuf[2], uint8_t *dest,
1165                                  int dstW, int yalpha, int uvalpha, int y)
1166 {
1167     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1168                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1169
1170     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1171     __asm__ volatile(
1172         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1173         "mov        %4, %%"REG_b"               \n\t"
1174         "push %%"REG_BP"                        \n\t"
1175         YSCALEYUV2RGB(%%REGBP, %5)
1176         "pxor    %%mm7, %%mm7                   \n\t"
1177         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1178 #ifdef DITHER1XBPP
1179         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1180         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1181         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1182 #endif
1183         WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1184         "pop %%"REG_BP"                         \n\t"
1185         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1186         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1187            "a" (&c->redDither)
1188     );
1189 }
1190
1191 static void RENAME(yuv2rgb565_2)(SwsContext *c, const int16_t *buf[2],
1192                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
1193                                  const int16_t *abuf[2], uint8_t *dest,
1194                                  int dstW, int yalpha, int uvalpha, int y)
1195 {
1196     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1197                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1198
1199     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1200     __asm__ volatile(
1201         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1202         "mov        %4, %%"REG_b"               \n\t"
1203         "push %%"REG_BP"                        \n\t"
1204         YSCALEYUV2RGB(%%REGBP, %5)
1205         "pxor    %%mm7, %%mm7                   \n\t"
1206         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1207 #ifdef DITHER1XBPP
1208         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1209         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1210         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1211 #endif
1212         WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1213         "pop %%"REG_BP"                         \n\t"
1214         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1215         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1216            "a" (&c->redDither)
1217     );
1218 }
1219
1220 #define REAL_YSCALEYUV2PACKED(index, c) \
1221     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
1222     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
1223     "psraw                $3, %%mm0                           \n\t"\
1224     "psraw                $3, %%mm1                           \n\t"\
1225     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1226     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1227     "xor            "#index", "#index"                        \n\t"\
1228     ".p2align              4            \n\t"\
1229     "1:                                 \n\t"\
1230     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1231     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1232     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1233     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1234     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1235     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1236     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
1237     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
1238     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
1239     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
1240     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
1241     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1242     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1243     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
1244     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
1245     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
1246     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
1247     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
1248     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
1249     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
1250     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
1251     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1252     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1253     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1254     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1255     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1256     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1257
1258 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
1259
1260 static void RENAME(yuv2yuyv422_2)(SwsContext *c, const int16_t *buf[2],
1261                                   const int16_t *ubuf[2], const int16_t *vbuf[2],
1262                                   const int16_t *abuf[2], uint8_t *dest,
1263                                   int dstW, int yalpha, int uvalpha, int y)
1264 {
1265     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1266                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1267
1268     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1269     __asm__ volatile(
1270         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1271         "mov %4, %%"REG_b"                        \n\t"
1272         "push %%"REG_BP"                        \n\t"
1273         YSCALEYUV2PACKED(%%REGBP, %5)
1274         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1275         "pop %%"REG_BP"                         \n\t"
1276         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1277         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1278            "a" (&c->redDither)
1279     );
1280 }
1281
1282 #define REAL_YSCALEYUV2RGB1(index, c) \
1283     "xor            "#index", "#index"  \n\t"\
1284     ".p2align              4            \n\t"\
1285     "1:                                 \n\t"\
1286     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1287     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1288     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1289     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1290     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1291     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1292     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1293     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1294     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1295     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1296     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1297     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1298     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1299     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1300     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1301     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1302     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1303     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1304     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1305     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1306     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1307     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1308     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1309     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1310     "paddw             %%mm3, %%mm4     \n\t"\
1311     "movq              %%mm2, %%mm0     \n\t"\
1312     "movq              %%mm5, %%mm6     \n\t"\
1313     "movq              %%mm4, %%mm3     \n\t"\
1314     "punpcklwd         %%mm2, %%mm2     \n\t"\
1315     "punpcklwd         %%mm5, %%mm5     \n\t"\
1316     "punpcklwd         %%mm4, %%mm4     \n\t"\
1317     "paddw             %%mm1, %%mm2     \n\t"\
1318     "paddw             %%mm1, %%mm5     \n\t"\
1319     "paddw             %%mm1, %%mm4     \n\t"\
1320     "punpckhwd         %%mm0, %%mm0     \n\t"\
1321     "punpckhwd         %%mm6, %%mm6     \n\t"\
1322     "punpckhwd         %%mm3, %%mm3     \n\t"\
1323     "paddw             %%mm7, %%mm0     \n\t"\
1324     "paddw             %%mm7, %%mm6     \n\t"\
1325     "paddw             %%mm7, %%mm3     \n\t"\
1326     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1327     "packuswb          %%mm0, %%mm2     \n\t"\
1328     "packuswb          %%mm6, %%mm5     \n\t"\
1329     "packuswb          %%mm3, %%mm4     \n\t"\
1330
1331 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
1332
1333 // do vertical chrominance interpolation
1334 #define REAL_YSCALEYUV2RGB1b(index, c) \
1335     "xor            "#index", "#index"  \n\t"\
1336     ".p2align              4            \n\t"\
1337     "1:                                 \n\t"\
1338     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1339     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1340     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1341     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1342     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1343     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1344     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1345     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1346     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
1347     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
1348     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1349     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1350     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1351     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1352     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1353     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1354     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1355     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1356     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1357     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1358     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1359     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1360     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1361     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1362     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1363     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1364     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1365     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1366     "paddw             %%mm3, %%mm4     \n\t"\
1367     "movq              %%mm2, %%mm0     \n\t"\
1368     "movq              %%mm5, %%mm6     \n\t"\
1369     "movq              %%mm4, %%mm3     \n\t"\
1370     "punpcklwd         %%mm2, %%mm2     \n\t"\
1371     "punpcklwd         %%mm5, %%mm5     \n\t"\
1372     "punpcklwd         %%mm4, %%mm4     \n\t"\
1373     "paddw             %%mm1, %%mm2     \n\t"\
1374     "paddw             %%mm1, %%mm5     \n\t"\
1375     "paddw             %%mm1, %%mm4     \n\t"\
1376     "punpckhwd         %%mm0, %%mm0     \n\t"\
1377     "punpckhwd         %%mm6, %%mm6     \n\t"\
1378     "punpckhwd         %%mm3, %%mm3     \n\t"\
1379     "paddw             %%mm7, %%mm0     \n\t"\
1380     "paddw             %%mm7, %%mm6     \n\t"\
1381     "paddw             %%mm7, %%mm3     \n\t"\
1382     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1383     "packuswb          %%mm0, %%mm2     \n\t"\
1384     "packuswb          %%mm6, %%mm5     \n\t"\
1385     "packuswb          %%mm3, %%mm4     \n\t"\
1386
1387 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
1388
1389 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
1390     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
1391     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
1392     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
1393     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
1394     "packuswb          %%mm1, %%mm7     \n\t"
1395 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
1396
1397 /**
1398  * YV12 to RGB without scaling or interpolating
1399  */
1400 static void RENAME(yuv2rgb32_1)(SwsContext *c, const int16_t *buf0,
1401                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1402                                 const int16_t *abuf0, uint8_t *dest,
1403                                 int dstW, int uvalpha, int y)
1404 {
1405     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1406     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1407
1408     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1409         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1410             __asm__ volatile(
1411                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1412                 "mov        %4, %%"REG_b"               \n\t"
1413                 "push %%"REG_BP"                        \n\t"
1414                 YSCALEYUV2RGB1(%%REGBP, %5)
1415                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1416                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1417                 "pop %%"REG_BP"                         \n\t"
1418                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1419                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1420                    "a" (&c->redDither)
1421             );
1422         } else {
1423             __asm__ volatile(
1424                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1425                 "mov        %4, %%"REG_b"               \n\t"
1426                 "push %%"REG_BP"                        \n\t"
1427                 YSCALEYUV2RGB1(%%REGBP, %5)
1428                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1429                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1430                 "pop %%"REG_BP"                         \n\t"
1431                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1432                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1433                    "a" (&c->redDither)
1434             );
1435         }
1436     } else {
1437         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1438             __asm__ volatile(
1439                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1440                 "mov        %4, %%"REG_b"               \n\t"
1441                 "push %%"REG_BP"                        \n\t"
1442                 YSCALEYUV2RGB1b(%%REGBP, %5)
1443                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1444                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1445                 "pop %%"REG_BP"                         \n\t"
1446                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1447                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1448                    "a" (&c->redDither)
1449             );
1450         } else {
1451             __asm__ volatile(
1452                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1453                 "mov        %4, %%"REG_b"               \n\t"
1454                 "push %%"REG_BP"                        \n\t"
1455                 YSCALEYUV2RGB1b(%%REGBP, %5)
1456                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1457                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1458                 "pop %%"REG_BP"                         \n\t"
1459                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1460                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1461                    "a" (&c->redDither)
1462             );
1463         }
1464     }
1465 }
1466
1467 static void RENAME(yuv2bgr24_1)(SwsContext *c, const int16_t *buf0,
1468                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1469                                 const int16_t *abuf0, uint8_t *dest,
1470                                 int dstW, int uvalpha, int y)
1471 {
1472     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1473     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1474
1475     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1476         __asm__ volatile(
1477             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1478             "mov        %4, %%"REG_b"               \n\t"
1479             "push %%"REG_BP"                        \n\t"
1480             YSCALEYUV2RGB1(%%REGBP, %5)
1481             "pxor    %%mm7, %%mm7                   \n\t"
1482             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1483             "pop %%"REG_BP"                         \n\t"
1484             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1485             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1486                "a" (&c->redDither)
1487         );
1488     } else {
1489         __asm__ volatile(
1490             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1491             "mov        %4, %%"REG_b"               \n\t"
1492             "push %%"REG_BP"                        \n\t"
1493             YSCALEYUV2RGB1b(%%REGBP, %5)
1494             "pxor    %%mm7, %%mm7                   \n\t"
1495             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1496             "pop %%"REG_BP"                         \n\t"
1497             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1498             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1499                "a" (&c->redDither)
1500         );
1501     }
1502 }
1503
1504 static void RENAME(yuv2rgb555_1)(SwsContext *c, const int16_t *buf0,
1505                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1506                                  const int16_t *abuf0, uint8_t *dest,
1507                                  int dstW, int uvalpha, int y)
1508 {
1509     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1510     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1511
1512     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1513         __asm__ volatile(
1514             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1515             "mov        %4, %%"REG_b"               \n\t"
1516             "push %%"REG_BP"                        \n\t"
1517             YSCALEYUV2RGB1(%%REGBP, %5)
1518             "pxor    %%mm7, %%mm7                   \n\t"
1519             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1520 #ifdef DITHER1XBPP
1521             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1522             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1523             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1524 #endif
1525             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1526             "pop %%"REG_BP"                         \n\t"
1527             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1528             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1529                "a" (&c->redDither)
1530         );
1531     } else {
1532         __asm__ volatile(
1533             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1534             "mov        %4, %%"REG_b"               \n\t"
1535             "push %%"REG_BP"                        \n\t"
1536             YSCALEYUV2RGB1b(%%REGBP, %5)
1537             "pxor    %%mm7, %%mm7                   \n\t"
1538             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1539 #ifdef DITHER1XBPP
1540             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1541             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1542             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1543 #endif
1544             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1545             "pop %%"REG_BP"                         \n\t"
1546             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1547             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1548                "a" (&c->redDither)
1549         );
1550     }
1551 }
1552
1553 static void RENAME(yuv2rgb565_1)(SwsContext *c, const int16_t *buf0,
1554                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1555                                  const int16_t *abuf0, uint8_t *dest,
1556                                  int dstW, int uvalpha, int y)
1557 {
1558     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1559     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1560
1561     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1562         __asm__ volatile(
1563             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1564             "mov        %4, %%"REG_b"               \n\t"
1565             "push %%"REG_BP"                        \n\t"
1566             YSCALEYUV2RGB1(%%REGBP, %5)
1567             "pxor    %%mm7, %%mm7                   \n\t"
1568             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1569 #ifdef DITHER1XBPP
1570             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1571             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1572             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1573 #endif
1574             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1575             "pop %%"REG_BP"                         \n\t"
1576             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1577             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1578                "a" (&c->redDither)
1579         );
1580     } else {
1581         __asm__ volatile(
1582             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1583             "mov        %4, %%"REG_b"               \n\t"
1584             "push %%"REG_BP"                        \n\t"
1585             YSCALEYUV2RGB1b(%%REGBP, %5)
1586             "pxor    %%mm7, %%mm7                   \n\t"
1587             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1588 #ifdef DITHER1XBPP
1589             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1590             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1591             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1592 #endif
1593             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1594             "pop %%"REG_BP"                         \n\t"
1595             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1596             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1597                "a" (&c->redDither)
1598         );
1599     }
1600 }
1601
1602 #define REAL_YSCALEYUV2PACKED1(index, c) \
1603     "xor            "#index", "#index"  \n\t"\
1604     ".p2align              4            \n\t"\
1605     "1:                                 \n\t"\
1606     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1607     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1608     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1609     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1610     "psraw                $7, %%mm3     \n\t" \
1611     "psraw                $7, %%mm4     \n\t" \
1612     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1613     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1614     "psraw                $7, %%mm1     \n\t" \
1615     "psraw                $7, %%mm7     \n\t" \
1616
1617 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
1618
1619 #define REAL_YSCALEYUV2PACKED1b(index, c) \
1620     "xor "#index", "#index"             \n\t"\
1621     ".p2align              4            \n\t"\
1622     "1:                                 \n\t"\
1623     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1624     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1625     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1626     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1627     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1628     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1629     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1630     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1631     "psrlw                $8, %%mm3     \n\t" \
1632     "psrlw                $8, %%mm4     \n\t" \
1633     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1634     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1635     "psraw                $7, %%mm1     \n\t" \
1636     "psraw                $7, %%mm7     \n\t"
1637 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
1638
1639 static void RENAME(yuv2yuyv422_1)(SwsContext *c, const int16_t *buf0,
1640                                   const int16_t *ubuf[2], const int16_t *bguf[2],
1641                                   const int16_t *abuf0, uint8_t *dest,
1642                                   int dstW, int uvalpha, int y)
1643 {
1644     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1645     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1646
1647     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1648         __asm__ volatile(
1649             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1650             "mov        %4, %%"REG_b"               \n\t"
1651             "push %%"REG_BP"                        \n\t"
1652             YSCALEYUV2PACKED1(%%REGBP, %5)
1653             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1654             "pop %%"REG_BP"                         \n\t"
1655             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1656             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1657                "a" (&c->redDither)
1658         );
1659     } else {
1660         __asm__ volatile(
1661             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1662             "mov        %4, %%"REG_b"               \n\t"
1663             "push %%"REG_BP"                        \n\t"
1664             YSCALEYUV2PACKED1b(%%REGBP, %5)
1665             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1666             "pop %%"REG_BP"                         \n\t"
1667             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1668             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1669                "a" (&c->redDither)
1670         );
1671     }
1672 }
1673
1674 #if !COMPILE_TEMPLATE_MMX2
1675 //FIXME yuy2* can read up to 7 samples too much
1676
1677 static void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src,
1678                             int width, uint32_t *unused)
1679 {
1680     __asm__ volatile(
1681         "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1682         "mov                    %0, %%"REG_a"       \n\t"
1683         "1:                                         \n\t"
1684         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1685         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1686         "pand                %%mm2, %%mm0           \n\t"
1687         "pand                %%mm2, %%mm1           \n\t"
1688         "packuswb            %%mm1, %%mm0           \n\t"
1689         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1690         "add                    $8, %%"REG_a"       \n\t"
1691         " js                    1b                  \n\t"
1692         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1693         : "%"REG_a
1694     );
1695 }
1696
1697 static void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV,
1698                              const uint8_t *src1, const uint8_t *src2,
1699                              int width, uint32_t *unused)
1700 {
1701     __asm__ volatile(
1702         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1703         "mov                    %0, %%"REG_a"       \n\t"
1704         "1:                                         \n\t"
1705         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1706         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1707         "psrlw                  $8, %%mm0           \n\t"
1708         "psrlw                  $8, %%mm1           \n\t"
1709         "packuswb            %%mm1, %%mm0           \n\t"
1710         "movq                %%mm0, %%mm1           \n\t"
1711         "psrlw                  $8, %%mm0           \n\t"
1712         "pand                %%mm4, %%mm1           \n\t"
1713         "packuswb            %%mm0, %%mm0           \n\t"
1714         "packuswb            %%mm1, %%mm1           \n\t"
1715         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1716         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1717         "add                    $4, %%"REG_a"       \n\t"
1718         " js                    1b                  \n\t"
1719         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1720         : "%"REG_a
1721     );
1722     assert(src1 == src2);
1723 }
1724
1725 /* This is almost identical to the previous, end exists only because
1726  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1727 static void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src,
1728                             int width, uint32_t *unused)
1729 {
1730     __asm__ volatile(
1731         "mov                  %0, %%"REG_a"         \n\t"
1732         "1:                                         \n\t"
1733         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1734         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1735         "psrlw                $8, %%mm0             \n\t"
1736         "psrlw                $8, %%mm1             \n\t"
1737         "packuswb          %%mm1, %%mm0             \n\t"
1738         "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1739         "add                  $8, %%"REG_a"         \n\t"
1740         " js                  1b                    \n\t"
1741         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1742         : "%"REG_a
1743     );
1744 }
1745
1746 static void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV,
1747                              const uint8_t *src1, const uint8_t *src2,
1748                              int width, uint32_t *unused)
1749 {
1750     __asm__ volatile(
1751         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1752         "mov                    %0, %%"REG_a"       \n\t"
1753         "1:                                         \n\t"
1754         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1755         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1756         "pand                %%mm4, %%mm0           \n\t"
1757         "pand                %%mm4, %%mm1           \n\t"
1758         "packuswb            %%mm1, %%mm0           \n\t"
1759         "movq                %%mm0, %%mm1           \n\t"
1760         "psrlw                  $8, %%mm0           \n\t"
1761         "pand                %%mm4, %%mm1           \n\t"
1762         "packuswb            %%mm0, %%mm0           \n\t"
1763         "packuswb            %%mm1, %%mm1           \n\t"
1764         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1765         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1766         "add                    $4, %%"REG_a"       \n\t"
1767         " js                    1b                  \n\t"
1768         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1769         : "%"REG_a
1770     );
1771     assert(src1 == src2);
1772 }
1773
1774 static av_always_inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1775                                               const uint8_t *src, int width)
1776 {
1777     __asm__ volatile(
1778         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1779         "mov                    %0, %%"REG_a"       \n\t"
1780         "1:                                         \n\t"
1781         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1782         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1783         "movq                %%mm0, %%mm2           \n\t"
1784         "movq                %%mm1, %%mm3           \n\t"
1785         "pand                %%mm4, %%mm0           \n\t"
1786         "pand                %%mm4, %%mm1           \n\t"
1787         "psrlw                  $8, %%mm2           \n\t"
1788         "psrlw                  $8, %%mm3           \n\t"
1789         "packuswb            %%mm1, %%mm0           \n\t"
1790         "packuswb            %%mm3, %%mm2           \n\t"
1791         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1792         "movq                %%mm2, (%3, %%"REG_a") \n\t"
1793         "add                    $8, %%"REG_a"       \n\t"
1794         " js                    1b                  \n\t"
1795         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1796         : "%"REG_a
1797     );
1798 }
1799
1800 static void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1801                              const uint8_t *src1, const uint8_t *src2,
1802                              int width, uint32_t *unused)
1803 {
1804     RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1805 }
1806
1807 static void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1808                              const uint8_t *src1, const uint8_t *src2,
1809                              int width, uint32_t *unused)
1810 {
1811     RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1812 }
1813 #endif /* !COMPILE_TEMPLATE_MMX2 */
1814
1815 static av_always_inline void RENAME(bgr24ToY_mmx)(int16_t *dst, const uint8_t *src,
1816                                                   int width, enum PixelFormat srcFormat)
1817 {
1818
1819     if(srcFormat == PIX_FMT_BGR24) {
1820         __asm__ volatile(
1821             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1822             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1823             :
1824         );
1825     } else {
1826         __asm__ volatile(
1827             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1828             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1829             :
1830         );
1831     }
1832
1833     __asm__ volatile(
1834         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1835         "mov                        %2, %%"REG_a"   \n\t"
1836         "pxor                    %%mm7, %%mm7       \n\t"
1837         "1:                                         \n\t"
1838         PREFETCH"               64(%0)              \n\t"
1839         "movd                     (%0), %%mm0       \n\t"
1840         "movd                    2(%0), %%mm1       \n\t"
1841         "movd                    6(%0), %%mm2       \n\t"
1842         "movd                    8(%0), %%mm3       \n\t"
1843         "add                       $12, %0          \n\t"
1844         "punpcklbw               %%mm7, %%mm0       \n\t"
1845         "punpcklbw               %%mm7, %%mm1       \n\t"
1846         "punpcklbw               %%mm7, %%mm2       \n\t"
1847         "punpcklbw               %%mm7, %%mm3       \n\t"
1848         "pmaddwd                 %%mm5, %%mm0       \n\t"
1849         "pmaddwd                 %%mm6, %%mm1       \n\t"
1850         "pmaddwd                 %%mm5, %%mm2       \n\t"
1851         "pmaddwd                 %%mm6, %%mm3       \n\t"
1852         "paddd                   %%mm1, %%mm0       \n\t"
1853         "paddd                   %%mm3, %%mm2       \n\t"
1854         "paddd                   %%mm4, %%mm0       \n\t"
1855         "paddd                   %%mm4, %%mm2       \n\t"
1856         "psrad                     $9, %%mm0       \n\t"
1857         "psrad                     $9, %%mm2       \n\t"
1858         "packssdw                %%mm2, %%mm0       \n\t"
1859         "movq                %%mm0, (%1, %%"REG_a") \n\t"
1860         "add                        $8, %%"REG_a"   \n\t"
1861         " js                        1b              \n\t"
1862     : "+r" (src)
1863     : "r" (dst+width), "g" ((x86_reg)-2*width)
1864     : "%"REG_a
1865     );
1866 }
1867
1868 static void RENAME(bgr24ToY)(int16_t *dst, const uint8_t *src,
1869                              int width, uint32_t *unused)
1870 {
1871     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1872 }
1873
1874 static void RENAME(rgb24ToY)(int16_t *dst, const uint8_t *src,
1875                              int width, uint32_t *unused)
1876 {
1877     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1878 }
1879
1880 static av_always_inline void RENAME(bgr24ToUV_mmx)(int16_t *dstU, int16_t *dstV,
1881                                                    const uint8_t *src, int width,
1882                                                    enum PixelFormat srcFormat)
1883 {
1884     __asm__ volatile(
1885         "movq                    24(%4), %%mm6       \n\t"
1886         "mov                        %3, %%"REG_a"   \n\t"
1887         "pxor                    %%mm7, %%mm7       \n\t"
1888         "1:                                         \n\t"
1889         PREFETCH"               64(%0)              \n\t"
1890         "movd                     (%0), %%mm0       \n\t"
1891         "movd                    2(%0), %%mm1       \n\t"
1892         "punpcklbw               %%mm7, %%mm0       \n\t"
1893         "punpcklbw               %%mm7, %%mm1       \n\t"
1894         "movq                    %%mm0, %%mm2       \n\t"
1895         "movq                    %%mm1, %%mm3       \n\t"
1896         "pmaddwd                  (%4), %%mm0       \n\t"
1897         "pmaddwd                 8(%4), %%mm1       \n\t"
1898         "pmaddwd                16(%4), %%mm2       \n\t"
1899         "pmaddwd                 %%mm6, %%mm3       \n\t"
1900         "paddd                   %%mm1, %%mm0       \n\t"
1901         "paddd                   %%mm3, %%mm2       \n\t"
1902
1903         "movd                    6(%0), %%mm1       \n\t"
1904         "movd                    8(%0), %%mm3       \n\t"
1905         "add                       $12, %0          \n\t"
1906         "punpcklbw               %%mm7, %%mm1       \n\t"
1907         "punpcklbw               %%mm7, %%mm3       \n\t"
1908         "movq                    %%mm1, %%mm4       \n\t"
1909         "movq                    %%mm3, %%mm5       \n\t"
1910         "pmaddwd                  (%4), %%mm1       \n\t"
1911         "pmaddwd                 8(%4), %%mm3       \n\t"
1912         "pmaddwd                16(%4), %%mm4       \n\t"
1913         "pmaddwd                 %%mm6, %%mm5       \n\t"
1914         "paddd                   %%mm3, %%mm1       \n\t"
1915         "paddd                   %%mm5, %%mm4       \n\t"
1916
1917         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1918         "paddd                   %%mm3, %%mm0       \n\t"
1919         "paddd                   %%mm3, %%mm2       \n\t"
1920         "paddd                   %%mm3, %%mm1       \n\t"
1921         "paddd                   %%mm3, %%mm4       \n\t"
1922         "psrad                     $9, %%mm0       \n\t"
1923         "psrad                     $9, %%mm2       \n\t"
1924         "psrad                     $9, %%mm1       \n\t"
1925         "psrad                     $9, %%mm4       \n\t"
1926         "packssdw                %%mm1, %%mm0       \n\t"
1927         "packssdw                %%mm4, %%mm2       \n\t"
1928         "movq                %%mm0, (%1, %%"REG_a") \n\t"
1929         "movq                %%mm2, (%2, %%"REG_a") \n\t"
1930         "add                        $8, %%"REG_a"   \n\t"
1931         " js                        1b              \n\t"
1932     : "+r" (src)
1933     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-2*width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1934     : "%"REG_a
1935     );
1936 }
1937
1938 static void RENAME(bgr24ToUV)(int16_t *dstU, int16_t *dstV,
1939                               const uint8_t *src1, const uint8_t *src2,
1940                               int width, uint32_t *unused)
1941 {
1942     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1943     assert(src1 == src2);
1944 }
1945
1946 static void RENAME(rgb24ToUV)(int16_t *dstU, int16_t *dstV,
1947                               const uint8_t *src1, const uint8_t *src2,
1948                               int width, uint32_t *unused)
1949 {
1950     assert(src1==src2);
1951     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1952 }
1953
1954 #if COMPILE_TEMPLATE_MMX2
1955 static void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
1956                                  int dstWidth, const uint8_t *src,
1957                                  int srcW, int xInc)
1958 {
1959     int16_t *filterPos = c->hLumFilterPos;
1960     int16_t *filter    = c->hLumFilter;
1961     void    *mmx2FilterCode= c->lumMmx2FilterCode;
1962     int i;
1963 #if defined(PIC)
1964     DECLARE_ALIGNED(8, uint64_t, ebxsave);
1965 #endif
1966
1967     __asm__ volatile(
1968 #if defined(PIC)
1969         "mov               %%"REG_b", %5        \n\t"
1970 #endif
1971         "pxor                  %%mm7, %%mm7     \n\t"
1972         "mov                      %0, %%"REG_c" \n\t"
1973         "mov                      %1, %%"REG_D" \n\t"
1974         "mov                      %2, %%"REG_d" \n\t"
1975         "mov                      %3, %%"REG_b" \n\t"
1976         "xor               %%"REG_a", %%"REG_a" \n\t" // i
1977         PREFETCH"        (%%"REG_c")            \n\t"
1978         PREFETCH"      32(%%"REG_c")            \n\t"
1979         PREFETCH"      64(%%"REG_c")            \n\t"
1980
1981 #if ARCH_X86_64
1982 #define CALL_MMX2_FILTER_CODE \
1983         "movl            (%%"REG_b"), %%esi     \n\t"\
1984         "call                    *%4            \n\t"\
1985         "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
1986         "add               %%"REG_S", %%"REG_c" \n\t"\
1987         "add               %%"REG_a", %%"REG_D" \n\t"\
1988         "xor               %%"REG_a", %%"REG_a" \n\t"\
1989
1990 #else
1991 #define CALL_MMX2_FILTER_CODE \
1992         "movl (%%"REG_b"), %%esi        \n\t"\
1993         "call         *%4                       \n\t"\
1994         "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
1995         "add               %%"REG_a", %%"REG_D" \n\t"\
1996         "xor               %%"REG_a", %%"REG_a" \n\t"\
1997
1998 #endif /* ARCH_X86_64 */
1999
2000         CALL_MMX2_FILTER_CODE
2001         CALL_MMX2_FILTER_CODE
2002         CALL_MMX2_FILTER_CODE
2003         CALL_MMX2_FILTER_CODE
2004         CALL_MMX2_FILTER_CODE
2005         CALL_MMX2_FILTER_CODE
2006         CALL_MMX2_FILTER_CODE
2007         CALL_MMX2_FILTER_CODE
2008
2009 #if defined(PIC)
2010         "mov                      %5, %%"REG_b" \n\t"
2011 #endif
2012         :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
2013            "m" (mmx2FilterCode)
2014 #if defined(PIC)
2015           ,"m" (ebxsave)
2016 #endif
2017         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2018 #if !defined(PIC)
2019          ,"%"REG_b
2020 #endif
2021     );
2022
2023     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
2024         dst[i] = src[srcW-1]*128;
2025 }
2026
2027 static void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2,
2028                                  int dstWidth, const uint8_t *src1,
2029                                  const uint8_t *src2, int srcW, int xInc)
2030 {
2031     int16_t *filterPos = c->hChrFilterPos;
2032     int16_t *filter    = c->hChrFilter;
2033     void    *mmx2FilterCode= c->chrMmx2FilterCode;
2034     int i;
2035 #if defined(PIC)
2036     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2037 #endif
2038
2039     __asm__ volatile(
2040 #if defined(PIC)
2041         "mov          %%"REG_b", %7         \n\t"
2042 #endif
2043         "pxor             %%mm7, %%mm7      \n\t"
2044         "mov                 %0, %%"REG_c"  \n\t"
2045         "mov                 %1, %%"REG_D"  \n\t"
2046         "mov                 %2, %%"REG_d"  \n\t"
2047         "mov                 %3, %%"REG_b"  \n\t"
2048         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2049         PREFETCH"   (%%"REG_c")             \n\t"
2050         PREFETCH" 32(%%"REG_c")             \n\t"
2051         PREFETCH" 64(%%"REG_c")             \n\t"
2052
2053         CALL_MMX2_FILTER_CODE
2054         CALL_MMX2_FILTER_CODE
2055         CALL_MMX2_FILTER_CODE
2056         CALL_MMX2_FILTER_CODE
2057         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2058         "mov                 %5, %%"REG_c"  \n\t" // src
2059         "mov                 %6, %%"REG_D"  \n\t" // buf2
2060         PREFETCH"   (%%"REG_c")             \n\t"
2061         PREFETCH" 32(%%"REG_c")             \n\t"
2062         PREFETCH" 64(%%"REG_c")             \n\t"
2063
2064         CALL_MMX2_FILTER_CODE
2065         CALL_MMX2_FILTER_CODE
2066         CALL_MMX2_FILTER_CODE
2067         CALL_MMX2_FILTER_CODE
2068
2069 #if defined(PIC)
2070         "mov %7, %%"REG_b"    \n\t"
2071 #endif
2072         :: "m" (src1), "m" (dst1), "m" (filter), "m" (filterPos),
2073            "m" (mmx2FilterCode), "m" (src2), "m"(dst2)
2074 #if defined(PIC)
2075           ,"m" (ebxsave)
2076 #endif
2077         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2078 #if !defined(PIC)
2079          ,"%"REG_b
2080 #endif
2081     );
2082
2083     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
2084         dst1[i] = src1[srcW-1]*128;
2085         dst2[i] = src2[srcW-1]*128;
2086     }
2087 }
2088 #endif /* COMPILE_TEMPLATE_MMX2 */
2089
2090 static av_cold void RENAME(sws_init_swScale)(SwsContext *c)
2091 {
2092     enum PixelFormat srcFormat = c->srcFormat,
2093                      dstFormat = c->dstFormat;
2094
2095     if (!is16BPS(dstFormat) && !is9_OR_10BPS(dstFormat) && dstFormat != PIX_FMT_NV12
2096         && dstFormat != PIX_FMT_NV21 && !(c->flags & SWS_BITEXACT)) {
2097             if (c->flags & SWS_ACCURATE_RND) {
2098                 c->yuv2yuv1 = RENAME(yuv2yuv1_ar    );
2099                 c->yuv2yuvX = RENAME(yuv2yuvX_ar    );
2100                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
2101                     switch (c->dstFormat) {
2102                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;
2103                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;
2104                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;
2105                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;
2106                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;
2107                     default: break;
2108                     }
2109                 }
2110             } else {
2111                 int should_dither= isNBPS(c->srcFormat) || is16BPS(c->srcFormat);
2112                 c->yuv2yuv1 = should_dither ? RENAME(yuv2yuv1_ar    ) : RENAME(yuv2yuv1    );
2113                 c->yuv2yuvX = RENAME(yuv2yuvX    );
2114                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
2115                     switch (c->dstFormat) {
2116                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;
2117                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;
2118                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;
2119                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;
2120                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;
2121                     default: break;
2122                     }
2123                 }
2124             }
2125         if (!(c->flags & SWS_FULL_CHR_H_INT)) {
2126             switch (c->dstFormat) {
2127             case PIX_FMT_RGB32:
2128                 c->yuv2packed1 = RENAME(yuv2rgb32_1);
2129                 c->yuv2packed2 = RENAME(yuv2rgb32_2);
2130                 break;
2131             case PIX_FMT_BGR24:
2132                 c->yuv2packed1 = RENAME(yuv2bgr24_1);
2133                 c->yuv2packed2 = RENAME(yuv2bgr24_2);
2134                 break;
2135             case PIX_FMT_RGB555:
2136                 c->yuv2packed1 = RENAME(yuv2rgb555_1);
2137                 c->yuv2packed2 = RENAME(yuv2rgb555_2);
2138                 break;
2139             case PIX_FMT_RGB565:
2140                 c->yuv2packed1 = RENAME(yuv2rgb565_1);
2141                 c->yuv2packed2 = RENAME(yuv2rgb565_2);
2142                 break;
2143             case PIX_FMT_YUYV422:
2144                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);
2145                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);
2146                 break;
2147             default:
2148                 break;
2149             }
2150         }
2151     }
2152
2153     if (c->srcBpc == 8 && c->dstBpc <= 10) {
2154     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2155 #if COMPILE_TEMPLATE_MMX2
2156     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
2157     {
2158         c->hyscale_fast = RENAME(hyscale_fast);
2159         c->hcscale_fast = RENAME(hcscale_fast);
2160     } else {
2161 #endif /* COMPILE_TEMPLATE_MMX2 */
2162         c->hyscale_fast = NULL;
2163         c->hcscale_fast = NULL;
2164 #if COMPILE_TEMPLATE_MMX2
2165     }
2166 #endif /* COMPILE_TEMPLATE_MMX2 */
2167     }
2168
2169 #if !COMPILE_TEMPLATE_MMX2
2170     switch(srcFormat) {
2171         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;
2172         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;
2173         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;
2174         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;
2175         default: break;
2176     }
2177 #endif /* !COMPILE_TEMPLATE_MMX2 */
2178     if (!c->chrSrcHSubSample) {
2179         switch(srcFormat) {
2180         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
2181         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
2182         default: break;
2183         }
2184     }
2185
2186     switch (srcFormat) {
2187 #if !COMPILE_TEMPLATE_MMX2
2188     case PIX_FMT_YUYV422  :
2189     case PIX_FMT_Y400A    : c->lumToYV12 = RENAME(yuy2ToY); break;
2190     case PIX_FMT_UYVY422  : c->lumToYV12 = RENAME(uyvyToY); break;
2191 #endif /* !COMPILE_TEMPLATE_MMX2 */
2192     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
2193     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
2194     default: break;
2195     }
2196 #if !COMPILE_TEMPLATE_MMX2
2197     if (c->alpPixBuf) {
2198         switch (srcFormat) {
2199         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;
2200         default: break;
2201         }
2202     }
2203 #endif /* !COMPILE_TEMPLATE_MMX2 */
2204 }