Merge remote-tracking branch 'qatar/master'
[ffmpeg.git] / libswscale / x86 / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #undef REAL_MOVNTQ
22 #undef MOVNTQ
23 #undef PREFETCH
24
25 #if COMPILE_TEMPLATE_MMX2
26 #define PREFETCH "prefetchnta"
27 #else
28 #define PREFETCH  " # nop"
29 #endif
30
31 #if COMPILE_TEMPLATE_MMX2
32 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
33 #else
34 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
35 #endif
36 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
37
38 #define YSCALEYUV2YV12X(offset, dest, end, pos) \
39     __asm__ volatile(\
40         "movq                  "DITHER16"+0(%0), %%mm3      \n\t"\
41         "movq                  "DITHER16"+8(%0), %%mm4      \n\t"\
42         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
43         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
44         ".p2align                             4             \n\t" /* FIXME Unroll? */\
45         "1:                                                 \n\t"\
46         "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\
47         "movq                (%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\
48         "movq               8(%%"REG_S", %3, 2), %%mm5      \n\t" /* srcData */\
49         "add                                $16, %%"REG_d"  \n\t"\
50         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
51         "test                         %%"REG_S", %%"REG_S"  \n\t"\
52         "pmulhw                           %%mm0, %%mm2      \n\t"\
53         "pmulhw                           %%mm0, %%mm5      \n\t"\
54         "paddw                            %%mm2, %%mm3      \n\t"\
55         "paddw                            %%mm5, %%mm4      \n\t"\
56         " jnz                                1b             \n\t"\
57         "psraw                               $3, %%mm3      \n\t"\
58         "psraw                               $3, %%mm4      \n\t"\
59         "packuswb                         %%mm4, %%mm3      \n\t"\
60         MOVNTQ(%%mm3, (%1, %3))\
61         "add                                 $8, %3         \n\t"\
62         "cmp                                 %2, %3         \n\t"\
63         "movq                  "DITHER16"+0(%0), %%mm3      \n\t"\
64         "movq                  "DITHER16"+8(%0), %%mm4      \n\t"\
65         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
66         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
67         "jb                                  1b             \n\t"\
68         :: "r" (&c->redDither),\
69            "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\
70         : "%"REG_d, "%"REG_S\
71     );
72
73 static void RENAME(yuv2yuvX)(SwsContext *c, const int16_t *lumFilter,
74                              const int16_t **lumSrc, int lumFilterSize,
75                              const int16_t *chrFilter, const int16_t **chrUSrc,
76                              const int16_t **chrVSrc,
77                              int chrFilterSize, const int16_t **alpSrc,
78                              uint8_t *dest[4], int dstW, int chrDstW,
79                              const uint8_t *lumDither, const uint8_t *chrDither)
80 {
81     int i;
82     uint8_t *yDest = dest[0], *uDest = dest[1], *vDest = dest[2],
83             *aDest = CONFIG_SWSCALE_ALPHA ? dest[3] : NULL;
84
85     if (uDest) {
86         x86_reg uv_off = c->uv_off;
87         for(i=0; i<8; i++) c->dither16[i] = chrDither[i]>>4;
88         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)
89         for(i=0; i<8; i++) c->dither16[i] = chrDither[(i+3)&7]>>4;
90         YSCALEYUV2YV12X(CHR_MMX_FILTER_OFFSET, vDest - uv_off, chrDstW + uv_off, uv_off)
91     }
92     for(i=0; i<8; i++) c->dither16[i] = lumDither[i]>>4;
93     if (CONFIG_SWSCALE_ALPHA && aDest) {
94         YSCALEYUV2YV12X(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)
95     }
96
97     YSCALEYUV2YV12X(LUM_MMX_FILTER_OFFSET, yDest, dstW, 0)
98 }
99
100 #define YSCALEYUV2YV12X_ACCURATE(offset, dest, end, pos) \
101     __asm__ volatile(\
102         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
103         "movq                  "DITHER32"+0(%0), %%mm4      \n\t"\
104         "movq                  "DITHER32"+8(%0), %%mm5      \n\t"\
105         "movq                 "DITHER32"+16(%0), %%mm6      \n\t"\
106         "movq                 "DITHER32"+24(%0), %%mm7      \n\t"\
107         "pxor                             %%mm4, %%mm4      \n\t"\
108         "pxor                             %%mm5, %%mm5      \n\t"\
109         "pxor                             %%mm6, %%mm6      \n\t"\
110         "pxor                             %%mm7, %%mm7      \n\t"\
111         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
112         ".p2align                             4             \n\t"\
113         "1:                                                 \n\t"\
114         "movq                (%%"REG_S", %3, 2), %%mm0      \n\t" /* srcData */\
115         "movq               8(%%"REG_S", %3, 2), %%mm2      \n\t" /* srcData */\
116         "mov        "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"  \n\t"\
117         "movq                (%%"REG_S", %3, 2), %%mm1      \n\t" /* srcData */\
118         "movq                             %%mm0, %%mm3      \n\t"\
119         "punpcklwd                        %%mm1, %%mm0      \n\t"\
120         "punpckhwd                        %%mm1, %%mm3      \n\t"\
121         "movq       "STR(APCK_COEF)"(%%"REG_d"), %%mm1      \n\t" /* filterCoeff */\
122         "pmaddwd                          %%mm1, %%mm0      \n\t"\
123         "pmaddwd                          %%mm1, %%mm3      \n\t"\
124         "paddd                            %%mm0, %%mm4      \n\t"\
125         "paddd                            %%mm3, %%mm5      \n\t"\
126         "movq               8(%%"REG_S", %3, 2), %%mm3      \n\t" /* srcData */\
127         "mov        "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"  \n\t"\
128         "add                  $"STR(APCK_SIZE)", %%"REG_d"  \n\t"\
129         "test                         %%"REG_S", %%"REG_S"  \n\t"\
130         "movq                             %%mm2, %%mm0      \n\t"\
131         "punpcklwd                        %%mm3, %%mm2      \n\t"\
132         "punpckhwd                        %%mm3, %%mm0      \n\t"\
133         "pmaddwd                          %%mm1, %%mm2      \n\t"\
134         "pmaddwd                          %%mm1, %%mm0      \n\t"\
135         "paddd                            %%mm2, %%mm6      \n\t"\
136         "paddd                            %%mm0, %%mm7      \n\t"\
137         " jnz                                1b             \n\t"\
138         "psrad                              $19, %%mm4      \n\t"\
139         "psrad                              $19, %%mm5      \n\t"\
140         "psrad                              $19, %%mm6      \n\t"\
141         "psrad                              $19, %%mm7      \n\t"\
142         "packssdw                         %%mm5, %%mm4      \n\t"\
143         "packssdw                         %%mm7, %%mm6      \n\t"\
144         "packuswb                         %%mm6, %%mm4      \n\t"\
145         MOVNTQ(%%mm4, (%1, %3))\
146         "add                                 $8, %3         \n\t"\
147         "cmp                                 %2, %3         \n\t"\
148         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
149         "movq                  "DITHER32"+0(%0), %%mm4      \n\t"\
150         "movq                  "DITHER32"+8(%0), %%mm5      \n\t"\
151         "movq                 "DITHER32"+16(%0), %%mm6      \n\t"\
152         "movq                 "DITHER32"+24(%0), %%mm7      \n\t"\
153         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
154         "jb                                  1b             \n\t"\
155         :: "r" (&c->redDither),\
156         "r" (dest), "g" ((x86_reg)(end)), "r"((x86_reg)(pos))\
157         : "%"REG_a, "%"REG_d, "%"REG_S\
158     );
159
160 static void RENAME(yuv2yuvX_ar)(SwsContext *c, const int16_t *lumFilter,
161                                 const int16_t **lumSrc, int lumFilterSize,
162                                 const int16_t *chrFilter, const int16_t **chrUSrc,
163                                 const int16_t **chrVSrc,
164                                 int chrFilterSize, const int16_t **alpSrc,
165                                 uint8_t *dest[4], int dstW, int chrDstW,
166                                 const uint8_t *lumDither, const uint8_t *chrDither)
167 {
168     int i;
169     uint8_t *yDest = dest[0], *uDest = dest[1], *vDest = dest[2],
170             *aDest = CONFIG_SWSCALE_ALPHA ? dest[3] : NULL;
171
172     if (uDest) {
173         x86_reg uv_off = c->uv_off;
174         for(i=0; i<8; i++) c->dither32[i] = chrDither[i]<<12;
175         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, uDest, chrDstW, 0)
176         for(i=0; i<8; i++) c->dither32[i] = chrDither[(i+3)&7]<<12;
177         YSCALEYUV2YV12X_ACCURATE(CHR_MMX_FILTER_OFFSET, vDest - uv_off, chrDstW + uv_off, uv_off)
178     }
179     for(i=0; i<8; i++) c->dither32[i] = lumDither[i]<<12;
180     if (CONFIG_SWSCALE_ALPHA && aDest) {
181         YSCALEYUV2YV12X_ACCURATE(ALP_MMX_FILTER_OFFSET, aDest, dstW, 0)
182     }
183
184     YSCALEYUV2YV12X_ACCURATE(LUM_MMX_FILTER_OFFSET, yDest, dstW, 0)
185 }
186
187 static void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc,
188                              const int16_t *chrUSrc, const int16_t *chrVSrc,
189                              const int16_t *alpSrc,
190                              uint8_t *dst[4], int dstW, int chrDstW,
191                              const uint8_t *lumDither, const uint8_t *chrDither)
192 {
193     int p= 4;
194     const int16_t *src[4]= {
195         lumSrc + dstW,     chrUSrc + chrDstW,
196         chrVSrc + chrDstW, alpSrc + dstW
197     };
198     x86_reg counter[4]= { dstW, chrDstW, chrDstW, dstW };
199
200     while (p--) {
201         if (dst[p]) {
202             __asm__ volatile(
203                 "mov %2, %%"REG_a"                    \n\t"
204                 ".p2align               4             \n\t" /* FIXME Unroll? */
205                 "1:                                   \n\t"
206                 "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"
207                 "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"
208                 "psraw                 $7, %%mm0      \n\t"
209                 "psraw                 $7, %%mm1      \n\t"
210                 "packuswb           %%mm1, %%mm0      \n\t"
211                 MOVNTQ(%%mm0, (%1, %%REGa))
212                 "add                   $8, %%"REG_a"  \n\t"
213                 "jnc                   1b             \n\t"
214                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
215                    "g" (-counter[p])
216                 : "%"REG_a
217             );
218         }
219     }
220 }
221
222 static void RENAME(yuv2yuv1_ar)(SwsContext *c, const int16_t *lumSrc,
223                                 const int16_t *chrUSrc, const int16_t *chrVSrc,
224                                 const int16_t *alpSrc,
225                                 uint8_t *dst[4], int dstW, int chrDstW,
226                                 const uint8_t *lumDither, const uint8_t *chrDither)
227 {
228     int p= 4;
229     const int16_t *src[4]= {
230         lumSrc + dstW,     chrUSrc + chrDstW,
231         chrVSrc + chrDstW, alpSrc + dstW
232     };
233     x86_reg counter[4]= { dstW, chrDstW, chrDstW, dstW };
234
235     while (p--) {
236         if (dst[p]) {
237             int i;
238             for(i=0; i<8; i++) c->dither16[i] = i<2 ? lumDither[i] : chrDither[i];
239             __asm__ volatile(
240                 "mov %2, %%"REG_a"                    \n\t"
241                 "movq               0(%3), %%mm6      \n\t"
242                 "movq               8(%3), %%mm7      \n\t"
243                 ".p2align                4            \n\t" /* FIXME Unroll? */
244                 "1:                                   \n\t"
245                 "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"
246                 "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"
247                 "paddsw             %%mm6, %%mm0      \n\t"
248                 "paddsw             %%mm7, %%mm1      \n\t"
249                 "psraw                 $7, %%mm0      \n\t"
250                 "psraw                 $7, %%mm1      \n\t"
251                 "packuswb           %%mm1, %%mm0      \n\t"
252                 MOVNTQ(%%mm0, (%1, %%REGa))
253                 "add                   $8, %%"REG_a"  \n\t"
254                 "jnc                   1b             \n\t"
255                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
256                    "g" (-counter[p]), "r"(c->dither16)
257                 : "%"REG_a
258             );
259         }
260     }
261 }
262
263 #define YSCALEYUV2PACKEDX_UV \
264     __asm__ volatile(\
265         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
266         ".p2align                      4                \n\t"\
267         "nop                                            \n\t"\
268         "1:                                             \n\t"\
269         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
270         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
271         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
272         "movq                      %%mm3, %%mm4         \n\t"\
273         ".p2align                      4                \n\t"\
274         "2:                                             \n\t"\
275         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
276         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
277         "add                          %6, %%"REG_S"     \n\t" \
278         "movq     (%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
279         "add                         $16, %%"REG_d"     \n\t"\
280         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
281         "pmulhw                    %%mm0, %%mm2         \n\t"\
282         "pmulhw                    %%mm0, %%mm5         \n\t"\
283         "paddw                     %%mm2, %%mm3         \n\t"\
284         "paddw                     %%mm5, %%mm4         \n\t"\
285         "test                  %%"REG_S", %%"REG_S"     \n\t"\
286         " jnz                         2b                \n\t"\
287
288 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
289     "lea                "offset"(%0), %%"REG_d"     \n\t"\
290     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
291     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
292     "movq                    "#dst1", "#dst2"       \n\t"\
293     ".p2align                      4                \n\t"\
294     "2:                                             \n\t"\
295     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
296     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
297     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
298     "add                         $16, %%"REG_d"            \n\t"\
299     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
300     "pmulhw                 "#coeff", "#src1"       \n\t"\
301     "pmulhw                 "#coeff", "#src2"       \n\t"\
302     "paddw                   "#src1", "#dst1"       \n\t"\
303     "paddw                   "#src2", "#dst2"       \n\t"\
304     "test                  %%"REG_S", %%"REG_S"     \n\t"\
305     " jnz                         2b                \n\t"\
306
307 #define YSCALEYUV2PACKEDX \
308     YSCALEYUV2PACKEDX_UV \
309     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
310
311 #define YSCALEYUV2PACKEDX_END                     \
312         :: "r" (&c->redDither),                   \
313             "m" (dummy), "m" (dummy), "m" (dummy),\
314             "r" (dest), "m" (dstW_reg), "m"(uv_off) \
315         : "%"REG_a, "%"REG_d, "%"REG_S            \
316     );
317
318 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
319     __asm__ volatile(\
320         "xor %%"REG_a", %%"REG_a"                       \n\t"\
321         ".p2align                      4                \n\t"\
322         "nop                                            \n\t"\
323         "1:                                             \n\t"\
324         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
325         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
326         "pxor                      %%mm4, %%mm4         \n\t"\
327         "pxor                      %%mm5, %%mm5         \n\t"\
328         "pxor                      %%mm6, %%mm6         \n\t"\
329         "pxor                      %%mm7, %%mm7         \n\t"\
330         ".p2align                      4                \n\t"\
331         "2:                                             \n\t"\
332         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
333         "add                          %6, %%"REG_S"      \n\t" \
334         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
335         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
336         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
337         "movq                      %%mm0, %%mm3         \n\t"\
338         "punpcklwd                 %%mm1, %%mm0         \n\t"\
339         "punpckhwd                 %%mm1, %%mm3         \n\t"\
340         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
341         "pmaddwd                   %%mm1, %%mm0         \n\t"\
342         "pmaddwd                   %%mm1, %%mm3         \n\t"\
343         "paddd                     %%mm0, %%mm4         \n\t"\
344         "paddd                     %%mm3, %%mm5         \n\t"\
345         "add                          %6, %%"REG_S"      \n\t" \
346         "movq     (%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
347         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
348         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
349         "test                  %%"REG_S", %%"REG_S"     \n\t"\
350         "movq                      %%mm2, %%mm0         \n\t"\
351         "punpcklwd                 %%mm3, %%mm2         \n\t"\
352         "punpckhwd                 %%mm3, %%mm0         \n\t"\
353         "pmaddwd                   %%mm1, %%mm2         \n\t"\
354         "pmaddwd                   %%mm1, %%mm0         \n\t"\
355         "paddd                     %%mm2, %%mm6         \n\t"\
356         "paddd                     %%mm0, %%mm7         \n\t"\
357         " jnz                         2b                \n\t"\
358         "psrad                       $16, %%mm4         \n\t"\
359         "psrad                       $16, %%mm5         \n\t"\
360         "psrad                       $16, %%mm6         \n\t"\
361         "psrad                       $16, %%mm7         \n\t"\
362         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
363         "packssdw                  %%mm5, %%mm4         \n\t"\
364         "packssdw                  %%mm7, %%mm6         \n\t"\
365         "paddw                     %%mm0, %%mm4         \n\t"\
366         "paddw                     %%mm0, %%mm6         \n\t"\
367         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
368         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
369
370 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
371     "lea                "offset"(%0), %%"REG_d"     \n\t"\
372     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
373     "pxor                      %%mm1, %%mm1         \n\t"\
374     "pxor                      %%mm5, %%mm5         \n\t"\
375     "pxor                      %%mm7, %%mm7         \n\t"\
376     "pxor                      %%mm6, %%mm6         \n\t"\
377     ".p2align                      4                \n\t"\
378     "2:                                             \n\t"\
379     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
380     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
381     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
382     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
383     "movq                      %%mm0, %%mm3         \n\t"\
384     "punpcklwd                 %%mm4, %%mm0         \n\t"\
385     "punpckhwd                 %%mm4, %%mm3         \n\t"\
386     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
387     "pmaddwd                   %%mm4, %%mm0         \n\t"\
388     "pmaddwd                   %%mm4, %%mm3         \n\t"\
389     "paddd                     %%mm0, %%mm1         \n\t"\
390     "paddd                     %%mm3, %%mm5         \n\t"\
391     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
392     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
393     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
394     "test                  %%"REG_S", %%"REG_S"     \n\t"\
395     "movq                      %%mm2, %%mm0         \n\t"\
396     "punpcklwd                 %%mm3, %%mm2         \n\t"\
397     "punpckhwd                 %%mm3, %%mm0         \n\t"\
398     "pmaddwd                   %%mm4, %%mm2         \n\t"\
399     "pmaddwd                   %%mm4, %%mm0         \n\t"\
400     "paddd                     %%mm2, %%mm7         \n\t"\
401     "paddd                     %%mm0, %%mm6         \n\t"\
402     " jnz                         2b                \n\t"\
403     "psrad                       $16, %%mm1         \n\t"\
404     "psrad                       $16, %%mm5         \n\t"\
405     "psrad                       $16, %%mm7         \n\t"\
406     "psrad                       $16, %%mm6         \n\t"\
407     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
408     "packssdw                  %%mm5, %%mm1         \n\t"\
409     "packssdw                  %%mm6, %%mm7         \n\t"\
410     "paddw                     %%mm0, %%mm1         \n\t"\
411     "paddw                     %%mm0, %%mm7         \n\t"\
412     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
413     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
414
415 #define YSCALEYUV2PACKEDX_ACCURATE \
416     YSCALEYUV2PACKEDX_ACCURATE_UV \
417     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
418
419 #define YSCALEYUV2RGBX \
420     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
421     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
422     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
423     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
424     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
425     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
426     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
427     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
428     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
429     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
430     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
431     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
432     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
433     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
434     "paddw           %%mm3, %%mm4       \n\t"\
435     "movq            %%mm2, %%mm0       \n\t"\
436     "movq            %%mm5, %%mm6       \n\t"\
437     "movq            %%mm4, %%mm3       \n\t"\
438     "punpcklwd       %%mm2, %%mm2       \n\t"\
439     "punpcklwd       %%mm5, %%mm5       \n\t"\
440     "punpcklwd       %%mm4, %%mm4       \n\t"\
441     "paddw           %%mm1, %%mm2       \n\t"\
442     "paddw           %%mm1, %%mm5       \n\t"\
443     "paddw           %%mm1, %%mm4       \n\t"\
444     "punpckhwd       %%mm0, %%mm0       \n\t"\
445     "punpckhwd       %%mm6, %%mm6       \n\t"\
446     "punpckhwd       %%mm3, %%mm3       \n\t"\
447     "paddw           %%mm7, %%mm0       \n\t"\
448     "paddw           %%mm7, %%mm6       \n\t"\
449     "paddw           %%mm7, %%mm3       \n\t"\
450     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
451     "packuswb        %%mm0, %%mm2       \n\t"\
452     "packuswb        %%mm6, %%mm5       \n\t"\
453     "packuswb        %%mm3, %%mm4       \n\t"\
454
455 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
456     "movq       "#b", "#q2"     \n\t" /* B */\
457     "movq       "#r", "#t"      \n\t" /* R */\
458     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
459     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
460     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
461     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
462     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
463     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
464     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
465     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
466     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
467     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
468 \
469     MOVNTQ(   q0,   (dst, index, 4))\
470     MOVNTQ(    b,  8(dst, index, 4))\
471     MOVNTQ(   q2, 16(dst, index, 4))\
472     MOVNTQ(   q3, 24(dst, index, 4))\
473 \
474     "add      $8, "#index"      \n\t"\
475     "cmp "#dstw", "#index"      \n\t"\
476     " jb      1b                \n\t"
477 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
478
479 static void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,
480                                    const int16_t **lumSrc, int lumFilterSize,
481                                    const int16_t *chrFilter, const int16_t **chrUSrc,
482                                    const int16_t **chrVSrc,
483                                    int chrFilterSize, const int16_t **alpSrc,
484                                    uint8_t *dest, int dstW, int dstY)
485 {
486     x86_reg dummy=0;
487     x86_reg dstW_reg = dstW;
488     x86_reg uv_off = c->uv_off << 1;
489
490     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
491         YSCALEYUV2PACKEDX_ACCURATE
492         YSCALEYUV2RGBX
493         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
494         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
495         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
496         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
497         "movq               "Y_TEMP"(%0), %%mm5         \n\t"
498         "psraw                        $3, %%mm1         \n\t"
499         "psraw                        $3, %%mm7         \n\t"
500         "packuswb                  %%mm7, %%mm1         \n\t"
501         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
502         YSCALEYUV2PACKEDX_END
503     } else {
504         YSCALEYUV2PACKEDX_ACCURATE
505         YSCALEYUV2RGBX
506         "pcmpeqd %%mm7, %%mm7 \n\t"
507         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
508         YSCALEYUV2PACKEDX_END
509     }
510 }
511
512 static void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,
513                                 const int16_t **lumSrc, int lumFilterSize,
514                                 const int16_t *chrFilter, const int16_t **chrUSrc,
515                                 const int16_t **chrVSrc,
516                                 int chrFilterSize, const int16_t **alpSrc,
517                                 uint8_t *dest, int dstW, int dstY)
518 {
519     x86_reg dummy=0;
520     x86_reg dstW_reg = dstW;
521     x86_reg uv_off = c->uv_off << 1;
522
523     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
524         YSCALEYUV2PACKEDX
525         YSCALEYUV2RGBX
526         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
527         "psraw                        $3, %%mm1         \n\t"
528         "psraw                        $3, %%mm7         \n\t"
529         "packuswb                  %%mm7, %%mm1         \n\t"
530         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
531         YSCALEYUV2PACKEDX_END
532     } else {
533         YSCALEYUV2PACKEDX
534         YSCALEYUV2RGBX
535         "pcmpeqd %%mm7, %%mm7 \n\t"
536         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
537         YSCALEYUV2PACKEDX_END
538     }
539 }
540
541 #define REAL_WRITERGB16(dst, dstw, index) \
542     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
543     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
544     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
545     "psrlq           $3, %%mm2  \n\t"\
546 \
547     "movq         %%mm2, %%mm1  \n\t"\
548     "movq         %%mm4, %%mm3  \n\t"\
549 \
550     "punpcklbw    %%mm7, %%mm3  \n\t"\
551     "punpcklbw    %%mm5, %%mm2  \n\t"\
552     "punpckhbw    %%mm7, %%mm4  \n\t"\
553     "punpckhbw    %%mm5, %%mm1  \n\t"\
554 \
555     "psllq           $3, %%mm3  \n\t"\
556     "psllq           $3, %%mm4  \n\t"\
557 \
558     "por          %%mm3, %%mm2  \n\t"\
559     "por          %%mm4, %%mm1  \n\t"\
560 \
561     MOVNTQ(%%mm2,  (dst, index, 2))\
562     MOVNTQ(%%mm1, 8(dst, index, 2))\
563 \
564     "add             $8, "#index"   \n\t"\
565     "cmp        "#dstw", "#index"   \n\t"\
566     " jb             1b             \n\t"
567 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
568
569 static void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,
570                                     const int16_t **lumSrc, int lumFilterSize,
571                                     const int16_t *chrFilter, const int16_t **chrUSrc,
572                                     const int16_t **chrVSrc,
573                                     int chrFilterSize, const int16_t **alpSrc,
574                                     uint8_t *dest, int dstW, int dstY)
575 {
576     x86_reg dummy=0;
577     x86_reg dstW_reg = dstW;
578     x86_reg uv_off = c->uv_off << 1;
579
580     YSCALEYUV2PACKEDX_ACCURATE
581     YSCALEYUV2RGBX
582     "pxor %%mm7, %%mm7 \n\t"
583     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
584 #ifdef DITHER1XBPP
585     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
586     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
587     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
588 #endif
589     WRITERGB16(%4, %5, %%REGa)
590     YSCALEYUV2PACKEDX_END
591 }
592
593 static void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,
594                                  const int16_t **lumSrc, int lumFilterSize,
595                                  const int16_t *chrFilter, const int16_t **chrUSrc,
596                                  const int16_t **chrVSrc,
597                                  int chrFilterSize, const int16_t **alpSrc,
598                                  uint8_t *dest, int dstW, int dstY)
599 {
600     x86_reg dummy=0;
601     x86_reg dstW_reg = dstW;
602     x86_reg uv_off = c->uv_off << 1;
603
604     YSCALEYUV2PACKEDX
605     YSCALEYUV2RGBX
606     "pxor %%mm7, %%mm7 \n\t"
607     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
608 #ifdef DITHER1XBPP
609     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
610     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
611     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
612 #endif
613     WRITERGB16(%4, %5, %%REGa)
614     YSCALEYUV2PACKEDX_END
615 }
616
617 #define REAL_WRITERGB15(dst, dstw, index) \
618     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
619     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
620     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
621     "psrlq           $3, %%mm2  \n\t"\
622     "psrlq           $1, %%mm5  \n\t"\
623 \
624     "movq         %%mm2, %%mm1  \n\t"\
625     "movq         %%mm4, %%mm3  \n\t"\
626 \
627     "punpcklbw    %%mm7, %%mm3  \n\t"\
628     "punpcklbw    %%mm5, %%mm2  \n\t"\
629     "punpckhbw    %%mm7, %%mm4  \n\t"\
630     "punpckhbw    %%mm5, %%mm1  \n\t"\
631 \
632     "psllq           $2, %%mm3  \n\t"\
633     "psllq           $2, %%mm4  \n\t"\
634 \
635     "por          %%mm3, %%mm2  \n\t"\
636     "por          %%mm4, %%mm1  \n\t"\
637 \
638     MOVNTQ(%%mm2,  (dst, index, 2))\
639     MOVNTQ(%%mm1, 8(dst, index, 2))\
640 \
641     "add             $8, "#index"   \n\t"\
642     "cmp        "#dstw", "#index"   \n\t"\
643     " jb             1b             \n\t"
644 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
645
646 static void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,
647                                     const int16_t **lumSrc, int lumFilterSize,
648                                     const int16_t *chrFilter, const int16_t **chrUSrc,
649                                     const int16_t **chrVSrc,
650                                     int chrFilterSize, const int16_t **alpSrc,
651                                     uint8_t *dest, int dstW, int dstY)
652 {
653     x86_reg dummy=0;
654     x86_reg dstW_reg = dstW;
655     x86_reg uv_off = c->uv_off << 1;
656
657     YSCALEYUV2PACKEDX_ACCURATE
658     YSCALEYUV2RGBX
659     "pxor %%mm7, %%mm7 \n\t"
660     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
661 #ifdef DITHER1XBPP
662     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
663     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
664     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
665 #endif
666     WRITERGB15(%4, %5, %%REGa)
667     YSCALEYUV2PACKEDX_END
668 }
669
670 static void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,
671                                  const int16_t **lumSrc, int lumFilterSize,
672                                  const int16_t *chrFilter, const int16_t **chrUSrc,
673                                  const int16_t **chrVSrc,
674                                  int chrFilterSize, const int16_t **alpSrc,
675                                  uint8_t *dest, int dstW, int dstY)
676 {
677     x86_reg dummy=0;
678     x86_reg dstW_reg = dstW;
679     x86_reg uv_off = c->uv_off << 1;
680
681     YSCALEYUV2PACKEDX
682     YSCALEYUV2RGBX
683     "pxor %%mm7, %%mm7 \n\t"
684     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
685 #ifdef DITHER1XBPP
686     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
687     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
688     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
689 #endif
690     WRITERGB15(%4, %5, %%REGa)
691     YSCALEYUV2PACKEDX_END
692 }
693
694 #define WRITEBGR24MMX(dst, dstw, index) \
695     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
696     "movq      %%mm2, %%mm1     \n\t" /* B */\
697     "movq      %%mm5, %%mm6     \n\t" /* R */\
698     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
699     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
700     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
701     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
702     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
703     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
704     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
705     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
706     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
707     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
708 \
709     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
710     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
711     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
712     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
713 \
714     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
715     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
716     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
717     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
718 \
719     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
720     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
721     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
722     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
723 \
724     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
725     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
726     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
727     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
728     MOVNTQ(%%mm0, (dst))\
729 \
730     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
731     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
732     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
733     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
734     MOVNTQ(%%mm6, 8(dst))\
735 \
736     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
737     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
738     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
739     MOVNTQ(%%mm5, 16(dst))\
740 \
741     "add         $24, "#dst"    \n\t"\
742 \
743     "add          $8, "#index"  \n\t"\
744     "cmp     "#dstw", "#index"  \n\t"\
745     " jb          1b            \n\t"
746
747 #define WRITEBGR24MMX2(dst, dstw, index) \
748     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
749     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
750     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
751     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
752     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
753     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
754 \
755     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
756     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
757     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
758 \
759     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
760     "por    %%mm1, %%mm6        \n\t"\
761     "por    %%mm3, %%mm6        \n\t"\
762     MOVNTQ(%%mm6, (dst))\
763 \
764     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
765     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
766     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
767     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
768 \
769     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
770     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
771     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
772 \
773     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
774     "por    %%mm3, %%mm6        \n\t"\
775     MOVNTQ(%%mm6, 8(dst))\
776 \
777     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
778     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
779     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
780 \
781     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
782     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
783     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
784 \
785     "por    %%mm1, %%mm3        \n\t"\
786     "por    %%mm3, %%mm6        \n\t"\
787     MOVNTQ(%%mm6, 16(dst))\
788 \
789     "add      $24, "#dst"       \n\t"\
790 \
791     "add       $8, "#index"     \n\t"\
792     "cmp  "#dstw", "#index"     \n\t"\
793     " jb       1b               \n\t"
794
795 #if COMPILE_TEMPLATE_MMX2
796 #undef WRITEBGR24
797 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
798 #else
799 #undef WRITEBGR24
800 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
801 #endif
802
803 static void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,
804                                    const int16_t **lumSrc, int lumFilterSize,
805                                    const int16_t *chrFilter, const int16_t **chrUSrc,
806                                    const int16_t **chrVSrc,
807                                    int chrFilterSize, const int16_t **alpSrc,
808                                    uint8_t *dest, int dstW, int dstY)
809 {
810     x86_reg dummy=0;
811     x86_reg dstW_reg = dstW;
812     x86_reg uv_off = c->uv_off << 1;
813
814     YSCALEYUV2PACKEDX_ACCURATE
815     YSCALEYUV2RGBX
816     "pxor %%mm7, %%mm7 \n\t"
817     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
818     "add %4, %%"REG_c"                        \n\t"
819     WRITEBGR24(%%REGc, %5, %%REGa)
820     :: "r" (&c->redDither),
821        "m" (dummy), "m" (dummy), "m" (dummy),
822        "r" (dest), "m" (dstW_reg), "m"(uv_off)
823     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
824     );
825 }
826
827 static void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,
828                                 const int16_t **lumSrc, int lumFilterSize,
829                                 const int16_t *chrFilter, const int16_t **chrUSrc,
830                                 const int16_t **chrVSrc,
831                                 int chrFilterSize, const int16_t **alpSrc,
832                                 uint8_t *dest, int dstW, int dstY)
833 {
834     x86_reg dummy=0;
835     x86_reg dstW_reg = dstW;
836     x86_reg uv_off = c->uv_off << 1;
837
838     YSCALEYUV2PACKEDX
839     YSCALEYUV2RGBX
840     "pxor                    %%mm7, %%mm7       \n\t"
841     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
842     "add                        %4, %%"REG_c"   \n\t"
843     WRITEBGR24(%%REGc, %5, %%REGa)
844     :: "r" (&c->redDither),
845        "m" (dummy), "m" (dummy), "m" (dummy),
846        "r" (dest),  "m" (dstW_reg), "m"(uv_off)
847     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
848     );
849 }
850
851 #define REAL_WRITEYUY2(dst, dstw, index) \
852     "packuswb  %%mm3, %%mm3     \n\t"\
853     "packuswb  %%mm4, %%mm4     \n\t"\
854     "packuswb  %%mm7, %%mm1     \n\t"\
855     "punpcklbw %%mm4, %%mm3     \n\t"\
856     "movq      %%mm1, %%mm7     \n\t"\
857     "punpcklbw %%mm3, %%mm1     \n\t"\
858     "punpckhbw %%mm3, %%mm7     \n\t"\
859 \
860     MOVNTQ(%%mm1, (dst, index, 2))\
861     MOVNTQ(%%mm7, 8(dst, index, 2))\
862 \
863     "add          $8, "#index"  \n\t"\
864     "cmp     "#dstw", "#index"  \n\t"\
865     " jb          1b            \n\t"
866 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
867
868 static void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,
869                                      const int16_t **lumSrc, int lumFilterSize,
870                                      const int16_t *chrFilter, const int16_t **chrUSrc,
871                                      const int16_t **chrVSrc,
872                                      int chrFilterSize, const int16_t **alpSrc,
873                                      uint8_t *dest, int dstW, int dstY)
874 {
875     x86_reg dummy=0;
876     x86_reg dstW_reg = dstW;
877     x86_reg uv_off = c->uv_off << 1;
878
879     YSCALEYUV2PACKEDX_ACCURATE
880     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
881     "psraw $3, %%mm3    \n\t"
882     "psraw $3, %%mm4    \n\t"
883     "psraw $3, %%mm1    \n\t"
884     "psraw $3, %%mm7    \n\t"
885     WRITEYUY2(%4, %5, %%REGa)
886     YSCALEYUV2PACKEDX_END
887 }
888
889 static void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,
890                                   const int16_t **lumSrc, int lumFilterSize,
891                                   const int16_t *chrFilter, const int16_t **chrUSrc,
892                                   const int16_t **chrVSrc,
893                                   int chrFilterSize, const int16_t **alpSrc,
894                                   uint8_t *dest, int dstW, int dstY)
895 {
896     x86_reg dummy=0;
897     x86_reg dstW_reg = dstW;
898     x86_reg uv_off = c->uv_off << 1;
899
900     YSCALEYUV2PACKEDX
901     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
902     "psraw $3, %%mm3    \n\t"
903     "psraw $3, %%mm4    \n\t"
904     "psraw $3, %%mm1    \n\t"
905     "psraw $3, %%mm7    \n\t"
906     WRITEYUY2(%4, %5, %%REGa)
907     YSCALEYUV2PACKEDX_END
908 }
909
910 #define REAL_YSCALEYUV2RGB_UV(index, c) \
911     "xor            "#index", "#index"  \n\t"\
912     ".p2align              4            \n\t"\
913     "1:                                 \n\t"\
914     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
915     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
916     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
917     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
918     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
919     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
920     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
921     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
922     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
923     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
924     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
925     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
926     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
927     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
928     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
929     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
930     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
931     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
932     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
933     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
934     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
935     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
936
937 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
938     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
939     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
940     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
941     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
942     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
943     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
944     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
945     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
946     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
947     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
948     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
949     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
950
951 #define REAL_YSCALEYUV2RGB_COEFF(c) \
952     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
953     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
954     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
955     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
956     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
957     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
958     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
959     "paddw             %%mm3, %%mm4     \n\t"\
960     "movq              %%mm2, %%mm0     \n\t"\
961     "movq              %%mm5, %%mm6     \n\t"\
962     "movq              %%mm4, %%mm3     \n\t"\
963     "punpcklwd         %%mm2, %%mm2     \n\t"\
964     "punpcklwd         %%mm5, %%mm5     \n\t"\
965     "punpcklwd         %%mm4, %%mm4     \n\t"\
966     "paddw             %%mm1, %%mm2     \n\t"\
967     "paddw             %%mm1, %%mm5     \n\t"\
968     "paddw             %%mm1, %%mm4     \n\t"\
969     "punpckhwd         %%mm0, %%mm0     \n\t"\
970     "punpckhwd         %%mm6, %%mm6     \n\t"\
971     "punpckhwd         %%mm3, %%mm3     \n\t"\
972     "paddw             %%mm7, %%mm0     \n\t"\
973     "paddw             %%mm7, %%mm6     \n\t"\
974     "paddw             %%mm7, %%mm3     \n\t"\
975     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
976     "packuswb          %%mm0, %%mm2     \n\t"\
977     "packuswb          %%mm6, %%mm5     \n\t"\
978     "packuswb          %%mm3, %%mm4     \n\t"\
979
980 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
981
982 #define YSCALEYUV2RGB(index, c) \
983     REAL_YSCALEYUV2RGB_UV(index, c) \
984     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
985     REAL_YSCALEYUV2RGB_COEFF(c)
986
987 /**
988  * vertical bilinear scale YV12 to RGB
989  */
990 static void RENAME(yuv2rgb32_2)(SwsContext *c, const int16_t *buf[2],
991                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
992                                 const int16_t *abuf[2], uint8_t *dest,
993                                 int dstW, int yalpha, int uvalpha, int y)
994 {
995     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
996                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
997
998     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
999         const int16_t *abuf0 = abuf[0], *abuf1 = abuf[1];
1000 #if ARCH_X86_64
1001         __asm__ volatile(
1002             YSCALEYUV2RGB(%%r8, %5)
1003             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
1004             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1005             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1006             "packuswb            %%mm7, %%mm1       \n\t"
1007             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1008             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "r" (dest),
1009                "a" (&c->redDither),
1010                "r" (abuf0), "r" (abuf1)
1011             : "%r8"
1012         );
1013 #else
1014         c->u_temp=(intptr_t)abuf0;
1015         c->v_temp=(intptr_t)abuf1;
1016         __asm__ volatile(
1017             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1018             "mov        %4, %%"REG_b"               \n\t"
1019             "push %%"REG_BP"                        \n\t"
1020             YSCALEYUV2RGB(%%REGBP, %5)
1021             "push                   %0              \n\t"
1022             "push                   %1              \n\t"
1023             "mov          "U_TEMP"(%5), %0          \n\t"
1024             "mov          "V_TEMP"(%5), %1          \n\t"
1025             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
1026             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1027             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1028             "packuswb            %%mm7, %%mm1       \n\t"
1029             "pop                    %1              \n\t"
1030             "pop                    %0              \n\t"
1031             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1032             "pop %%"REG_BP"                         \n\t"
1033             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1034             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1035                "a" (&c->redDither)
1036         );
1037 #endif
1038     } else {
1039         __asm__ volatile(
1040             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1041             "mov        %4, %%"REG_b"               \n\t"
1042             "push %%"REG_BP"                        \n\t"
1043             YSCALEYUV2RGB(%%REGBP, %5)
1044             "pcmpeqd %%mm7, %%mm7                   \n\t"
1045             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1046             "pop %%"REG_BP"                         \n\t"
1047             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1048             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1049                "a" (&c->redDither)
1050         );
1051     }
1052 }
1053
1054 static void RENAME(yuv2bgr24_2)(SwsContext *c, const int16_t *buf[2],
1055                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
1056                                 const int16_t *abuf[2], uint8_t *dest,
1057                                 int dstW, int yalpha, int uvalpha, int y)
1058 {
1059     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1060                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1061
1062     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1063     __asm__ volatile(
1064         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1065         "mov        %4, %%"REG_b"               \n\t"
1066         "push %%"REG_BP"                        \n\t"
1067         YSCALEYUV2RGB(%%REGBP, %5)
1068         "pxor    %%mm7, %%mm7                   \n\t"
1069         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1070         "pop %%"REG_BP"                         \n\t"
1071         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1072         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1073            "a" (&c->redDither)
1074     );
1075 }
1076
1077 static void RENAME(yuv2rgb555_2)(SwsContext *c, const int16_t *buf[2],
1078                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
1079                                  const int16_t *abuf[2], uint8_t *dest,
1080                                  int dstW, int yalpha, int uvalpha, int y)
1081 {
1082     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1083                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1084
1085     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1086     __asm__ volatile(
1087         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1088         "mov        %4, %%"REG_b"               \n\t"
1089         "push %%"REG_BP"                        \n\t"
1090         YSCALEYUV2RGB(%%REGBP, %5)
1091         "pxor    %%mm7, %%mm7                   \n\t"
1092         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1093 #ifdef DITHER1XBPP
1094         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1095         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1096         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1097 #endif
1098         WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1099         "pop %%"REG_BP"                         \n\t"
1100         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1101         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1102            "a" (&c->redDither)
1103     );
1104 }
1105
1106 static void RENAME(yuv2rgb565_2)(SwsContext *c, const int16_t *buf[2],
1107                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
1108                                  const int16_t *abuf[2], uint8_t *dest,
1109                                  int dstW, int yalpha, int uvalpha, int y)
1110 {
1111     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1112                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1113
1114     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1115     __asm__ volatile(
1116         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1117         "mov        %4, %%"REG_b"               \n\t"
1118         "push %%"REG_BP"                        \n\t"
1119         YSCALEYUV2RGB(%%REGBP, %5)
1120         "pxor    %%mm7, %%mm7                   \n\t"
1121         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1122 #ifdef DITHER1XBPP
1123         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1124         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1125         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1126 #endif
1127         WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1128         "pop %%"REG_BP"                         \n\t"
1129         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1130         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1131            "a" (&c->redDither)
1132     );
1133 }
1134
1135 #define REAL_YSCALEYUV2PACKED(index, c) \
1136     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
1137     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
1138     "psraw                $3, %%mm0                           \n\t"\
1139     "psraw                $3, %%mm1                           \n\t"\
1140     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1141     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1142     "xor            "#index", "#index"                        \n\t"\
1143     ".p2align              4            \n\t"\
1144     "1:                                 \n\t"\
1145     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1146     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1147     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1148     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1149     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1150     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1151     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
1152     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
1153     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
1154     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
1155     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
1156     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1157     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1158     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
1159     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
1160     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
1161     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
1162     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
1163     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
1164     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
1165     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
1166     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1167     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1168     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1169     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1170     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1171     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1172
1173 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
1174
1175 static void RENAME(yuv2yuyv422_2)(SwsContext *c, const int16_t *buf[2],
1176                                   const int16_t *ubuf[2], const int16_t *vbuf[2],
1177                                   const int16_t *abuf[2], uint8_t *dest,
1178                                   int dstW, int yalpha, int uvalpha, int y)
1179 {
1180     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1181                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1182
1183     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1184     __asm__ volatile(
1185         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1186         "mov %4, %%"REG_b"                        \n\t"
1187         "push %%"REG_BP"                        \n\t"
1188         YSCALEYUV2PACKED(%%REGBP, %5)
1189         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1190         "pop %%"REG_BP"                         \n\t"
1191         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1192         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1193            "a" (&c->redDither)
1194     );
1195 }
1196
1197 #define REAL_YSCALEYUV2RGB1(index, c) \
1198     "xor            "#index", "#index"  \n\t"\
1199     ".p2align              4            \n\t"\
1200     "1:                                 \n\t"\
1201     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1202     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1203     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1204     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1205     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1206     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1207     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1208     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1209     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1210     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1211     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1212     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1213     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1214     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1215     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1216     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1217     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1218     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1219     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1220     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1221     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1222     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1223     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1224     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1225     "paddw             %%mm3, %%mm4     \n\t"\
1226     "movq              %%mm2, %%mm0     \n\t"\
1227     "movq              %%mm5, %%mm6     \n\t"\
1228     "movq              %%mm4, %%mm3     \n\t"\
1229     "punpcklwd         %%mm2, %%mm2     \n\t"\
1230     "punpcklwd         %%mm5, %%mm5     \n\t"\
1231     "punpcklwd         %%mm4, %%mm4     \n\t"\
1232     "paddw             %%mm1, %%mm2     \n\t"\
1233     "paddw             %%mm1, %%mm5     \n\t"\
1234     "paddw             %%mm1, %%mm4     \n\t"\
1235     "punpckhwd         %%mm0, %%mm0     \n\t"\
1236     "punpckhwd         %%mm6, %%mm6     \n\t"\
1237     "punpckhwd         %%mm3, %%mm3     \n\t"\
1238     "paddw             %%mm7, %%mm0     \n\t"\
1239     "paddw             %%mm7, %%mm6     \n\t"\
1240     "paddw             %%mm7, %%mm3     \n\t"\
1241     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1242     "packuswb          %%mm0, %%mm2     \n\t"\
1243     "packuswb          %%mm6, %%mm5     \n\t"\
1244     "packuswb          %%mm3, %%mm4     \n\t"\
1245
1246 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
1247
1248 // do vertical chrominance interpolation
1249 #define REAL_YSCALEYUV2RGB1b(index, c) \
1250     "xor            "#index", "#index"  \n\t"\
1251     ".p2align              4            \n\t"\
1252     "1:                                 \n\t"\
1253     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1254     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1255     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1256     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1257     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1258     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1259     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1260     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1261     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
1262     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
1263     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1264     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1265     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1266     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1267     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1268     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1269     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1270     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1271     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1272     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1273     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1274     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1275     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1276     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1277     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1278     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1279     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1280     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1281     "paddw             %%mm3, %%mm4     \n\t"\
1282     "movq              %%mm2, %%mm0     \n\t"\
1283     "movq              %%mm5, %%mm6     \n\t"\
1284     "movq              %%mm4, %%mm3     \n\t"\
1285     "punpcklwd         %%mm2, %%mm2     \n\t"\
1286     "punpcklwd         %%mm5, %%mm5     \n\t"\
1287     "punpcklwd         %%mm4, %%mm4     \n\t"\
1288     "paddw             %%mm1, %%mm2     \n\t"\
1289     "paddw             %%mm1, %%mm5     \n\t"\
1290     "paddw             %%mm1, %%mm4     \n\t"\
1291     "punpckhwd         %%mm0, %%mm0     \n\t"\
1292     "punpckhwd         %%mm6, %%mm6     \n\t"\
1293     "punpckhwd         %%mm3, %%mm3     \n\t"\
1294     "paddw             %%mm7, %%mm0     \n\t"\
1295     "paddw             %%mm7, %%mm6     \n\t"\
1296     "paddw             %%mm7, %%mm3     \n\t"\
1297     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1298     "packuswb          %%mm0, %%mm2     \n\t"\
1299     "packuswb          %%mm6, %%mm5     \n\t"\
1300     "packuswb          %%mm3, %%mm4     \n\t"\
1301
1302 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
1303
1304 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
1305     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
1306     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
1307     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
1308     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
1309     "packuswb          %%mm1, %%mm7     \n\t"
1310 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
1311
1312 /**
1313  * YV12 to RGB without scaling or interpolating
1314  */
1315 static void RENAME(yuv2rgb32_1)(SwsContext *c, const int16_t *buf0,
1316                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1317                                 const int16_t *abuf0, uint8_t *dest,
1318                                 int dstW, int uvalpha, int y)
1319 {
1320     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1321     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1322
1323     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1324         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1325             __asm__ volatile(
1326                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1327                 "mov        %4, %%"REG_b"               \n\t"
1328                 "push %%"REG_BP"                        \n\t"
1329                 YSCALEYUV2RGB1(%%REGBP, %5)
1330                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1331                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1332                 "pop %%"REG_BP"                         \n\t"
1333                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1334                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1335                    "a" (&c->redDither)
1336             );
1337         } else {
1338             __asm__ volatile(
1339                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1340                 "mov        %4, %%"REG_b"               \n\t"
1341                 "push %%"REG_BP"                        \n\t"
1342                 YSCALEYUV2RGB1(%%REGBP, %5)
1343                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1344                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1345                 "pop %%"REG_BP"                         \n\t"
1346                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1347                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1348                    "a" (&c->redDither)
1349             );
1350         }
1351     } else {
1352         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1353             __asm__ volatile(
1354                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1355                 "mov        %4, %%"REG_b"               \n\t"
1356                 "push %%"REG_BP"                        \n\t"
1357                 YSCALEYUV2RGB1b(%%REGBP, %5)
1358                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1359                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1360                 "pop %%"REG_BP"                         \n\t"
1361                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1362                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1363                    "a" (&c->redDither)
1364             );
1365         } else {
1366             __asm__ volatile(
1367                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1368                 "mov        %4, %%"REG_b"               \n\t"
1369                 "push %%"REG_BP"                        \n\t"
1370                 YSCALEYUV2RGB1b(%%REGBP, %5)
1371                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1372                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1373                 "pop %%"REG_BP"                         \n\t"
1374                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1375                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1376                    "a" (&c->redDither)
1377             );
1378         }
1379     }
1380 }
1381
1382 static void RENAME(yuv2bgr24_1)(SwsContext *c, const int16_t *buf0,
1383                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1384                                 const int16_t *abuf0, uint8_t *dest,
1385                                 int dstW, int uvalpha, int y)
1386 {
1387     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1388     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1389
1390     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1391         __asm__ volatile(
1392             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1393             "mov        %4, %%"REG_b"               \n\t"
1394             "push %%"REG_BP"                        \n\t"
1395             YSCALEYUV2RGB1(%%REGBP, %5)
1396             "pxor    %%mm7, %%mm7                   \n\t"
1397             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1398             "pop %%"REG_BP"                         \n\t"
1399             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1400             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1401                "a" (&c->redDither)
1402         );
1403     } else {
1404         __asm__ volatile(
1405             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1406             "mov        %4, %%"REG_b"               \n\t"
1407             "push %%"REG_BP"                        \n\t"
1408             YSCALEYUV2RGB1b(%%REGBP, %5)
1409             "pxor    %%mm7, %%mm7                   \n\t"
1410             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1411             "pop %%"REG_BP"                         \n\t"
1412             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1413             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1414                "a" (&c->redDither)
1415         );
1416     }
1417 }
1418
1419 static void RENAME(yuv2rgb555_1)(SwsContext *c, const int16_t *buf0,
1420                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1421                                  const int16_t *abuf0, uint8_t *dest,
1422                                  int dstW, int uvalpha, int y)
1423 {
1424     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1425     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1426
1427     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1428         __asm__ volatile(
1429             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1430             "mov        %4, %%"REG_b"               \n\t"
1431             "push %%"REG_BP"                        \n\t"
1432             YSCALEYUV2RGB1(%%REGBP, %5)
1433             "pxor    %%mm7, %%mm7                   \n\t"
1434             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1435 #ifdef DITHER1XBPP
1436             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1437             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1438             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1439 #endif
1440             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1441             "pop %%"REG_BP"                         \n\t"
1442             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1443             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1444                "a" (&c->redDither)
1445         );
1446     } else {
1447         __asm__ volatile(
1448             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1449             "mov        %4, %%"REG_b"               \n\t"
1450             "push %%"REG_BP"                        \n\t"
1451             YSCALEYUV2RGB1b(%%REGBP, %5)
1452             "pxor    %%mm7, %%mm7                   \n\t"
1453             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1454 #ifdef DITHER1XBPP
1455             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1456             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1457             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1458 #endif
1459             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1460             "pop %%"REG_BP"                         \n\t"
1461             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1462             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1463                "a" (&c->redDither)
1464         );
1465     }
1466 }
1467
1468 static void RENAME(yuv2rgb565_1)(SwsContext *c, const int16_t *buf0,
1469                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1470                                  const int16_t *abuf0, uint8_t *dest,
1471                                  int dstW, int uvalpha, int y)
1472 {
1473     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1474     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1475
1476     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1477         __asm__ volatile(
1478             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1479             "mov        %4, %%"REG_b"               \n\t"
1480             "push %%"REG_BP"                        \n\t"
1481             YSCALEYUV2RGB1(%%REGBP, %5)
1482             "pxor    %%mm7, %%mm7                   \n\t"
1483             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1484 #ifdef DITHER1XBPP
1485             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1486             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1487             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1488 #endif
1489             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1490             "pop %%"REG_BP"                         \n\t"
1491             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1492             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1493                "a" (&c->redDither)
1494         );
1495     } else {
1496         __asm__ volatile(
1497             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1498             "mov        %4, %%"REG_b"               \n\t"
1499             "push %%"REG_BP"                        \n\t"
1500             YSCALEYUV2RGB1b(%%REGBP, %5)
1501             "pxor    %%mm7, %%mm7                   \n\t"
1502             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1503 #ifdef DITHER1XBPP
1504             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1505             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1506             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1507 #endif
1508             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1509             "pop %%"REG_BP"                         \n\t"
1510             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1511             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1512                "a" (&c->redDither)
1513         );
1514     }
1515 }
1516
1517 #define REAL_YSCALEYUV2PACKED1(index, c) \
1518     "xor            "#index", "#index"  \n\t"\
1519     ".p2align              4            \n\t"\
1520     "1:                                 \n\t"\
1521     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1522     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1523     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1524     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1525     "psraw                $7, %%mm3     \n\t" \
1526     "psraw                $7, %%mm4     \n\t" \
1527     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1528     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1529     "psraw                $7, %%mm1     \n\t" \
1530     "psraw                $7, %%mm7     \n\t" \
1531
1532 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
1533
1534 #define REAL_YSCALEYUV2PACKED1b(index, c) \
1535     "xor "#index", "#index"             \n\t"\
1536     ".p2align              4            \n\t"\
1537     "1:                                 \n\t"\
1538     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1539     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1540     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1541     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1542     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1543     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1544     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1545     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1546     "psrlw                $8, %%mm3     \n\t" \
1547     "psrlw                $8, %%mm4     \n\t" \
1548     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1549     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1550     "psraw                $7, %%mm1     \n\t" \
1551     "psraw                $7, %%mm7     \n\t"
1552 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
1553
1554 static void RENAME(yuv2yuyv422_1)(SwsContext *c, const int16_t *buf0,
1555                                   const int16_t *ubuf[2], const int16_t *bguf[2],
1556                                   const int16_t *abuf0, uint8_t *dest,
1557                                   int dstW, int uvalpha, int y)
1558 {
1559     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1560     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1561
1562     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1563         __asm__ volatile(
1564             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1565             "mov        %4, %%"REG_b"               \n\t"
1566             "push %%"REG_BP"                        \n\t"
1567             YSCALEYUV2PACKED1(%%REGBP, %5)
1568             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1569             "pop %%"REG_BP"                         \n\t"
1570             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1571             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1572                "a" (&c->redDither)
1573         );
1574     } else {
1575         __asm__ volatile(
1576             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1577             "mov        %4, %%"REG_b"               \n\t"
1578             "push %%"REG_BP"                        \n\t"
1579             YSCALEYUV2PACKED1b(%%REGBP, %5)
1580             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1581             "pop %%"REG_BP"                         \n\t"
1582             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1583             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1584                "a" (&c->redDither)
1585         );
1586     }
1587 }
1588
1589 #if !COMPILE_TEMPLATE_MMX2
1590 //FIXME yuy2* can read up to 7 samples too much
1591
1592 static void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src,
1593                             int width, uint32_t *unused)
1594 {
1595     __asm__ volatile(
1596         "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1597         "mov                    %0, %%"REG_a"       \n\t"
1598         "1:                                         \n\t"
1599         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1600         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1601         "pand                %%mm2, %%mm0           \n\t"
1602         "pand                %%mm2, %%mm1           \n\t"
1603         "packuswb            %%mm1, %%mm0           \n\t"
1604         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1605         "add                    $8, %%"REG_a"       \n\t"
1606         " js                    1b                  \n\t"
1607         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1608         : "%"REG_a
1609     );
1610 }
1611
1612 static void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV,
1613                              const uint8_t *src1, const uint8_t *src2,
1614                              int width, uint32_t *unused)
1615 {
1616     __asm__ volatile(
1617         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1618         "mov                    %0, %%"REG_a"       \n\t"
1619         "1:                                         \n\t"
1620         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1621         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1622         "psrlw                  $8, %%mm0           \n\t"
1623         "psrlw                  $8, %%mm1           \n\t"
1624         "packuswb            %%mm1, %%mm0           \n\t"
1625         "movq                %%mm0, %%mm1           \n\t"
1626         "psrlw                  $8, %%mm0           \n\t"
1627         "pand                %%mm4, %%mm1           \n\t"
1628         "packuswb            %%mm0, %%mm0           \n\t"
1629         "packuswb            %%mm1, %%mm1           \n\t"
1630         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1631         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1632         "add                    $4, %%"REG_a"       \n\t"
1633         " js                    1b                  \n\t"
1634         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1635         : "%"REG_a
1636     );
1637     assert(src1 == src2);
1638 }
1639
1640 static void RENAME(LEToUV)(uint8_t *dstU, uint8_t *dstV,
1641                            const uint8_t *src1, const uint8_t *src2,
1642                            int width, uint32_t *unused)
1643 {
1644     __asm__ volatile(
1645         "mov                    %0, %%"REG_a"       \n\t"
1646         "1:                                         \n\t"
1647         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1648         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1649         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1650         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1651         "psrlw                  $8, %%mm0           \n\t"
1652         "psrlw                  $8, %%mm1           \n\t"
1653         "psrlw                  $8, %%mm2           \n\t"
1654         "psrlw                  $8, %%mm3           \n\t"
1655         "packuswb            %%mm1, %%mm0           \n\t"
1656         "packuswb            %%mm3, %%mm2           \n\t"
1657         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1658         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1659         "add                    $8, %%"REG_a"       \n\t"
1660         " js                    1b                  \n\t"
1661         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1662         : "%"REG_a
1663     );
1664 }
1665
1666 /* This is almost identical to the previous, end exists only because
1667  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1668 static void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src,
1669                             int width, uint32_t *unused)
1670 {
1671     __asm__ volatile(
1672         "mov                  %0, %%"REG_a"         \n\t"
1673         "1:                                         \n\t"
1674         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1675         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1676         "psrlw                $8, %%mm0             \n\t"
1677         "psrlw                $8, %%mm1             \n\t"
1678         "packuswb          %%mm1, %%mm0             \n\t"
1679         "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1680         "add                  $8, %%"REG_a"         \n\t"
1681         " js                  1b                    \n\t"
1682         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1683         : "%"REG_a
1684     );
1685 }
1686
1687 static void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV,
1688                              const uint8_t *src1, const uint8_t *src2,
1689                              int width, uint32_t *unused)
1690 {
1691     __asm__ volatile(
1692         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1693         "mov                    %0, %%"REG_a"       \n\t"
1694         "1:                                         \n\t"
1695         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1696         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1697         "pand                %%mm4, %%mm0           \n\t"
1698         "pand                %%mm4, %%mm1           \n\t"
1699         "packuswb            %%mm1, %%mm0           \n\t"
1700         "movq                %%mm0, %%mm1           \n\t"
1701         "psrlw                  $8, %%mm0           \n\t"
1702         "pand                %%mm4, %%mm1           \n\t"
1703         "packuswb            %%mm0, %%mm0           \n\t"
1704         "packuswb            %%mm1, %%mm1           \n\t"
1705         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1706         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1707         "add                    $4, %%"REG_a"       \n\t"
1708         " js                    1b                  \n\t"
1709         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1710         : "%"REG_a
1711     );
1712     assert(src1 == src2);
1713 }
1714
1715 static void RENAME(BEToUV)(uint8_t *dstU, uint8_t *dstV,
1716                            const uint8_t *src1, const uint8_t *src2,
1717                            int width, uint32_t *unused)
1718 {
1719     __asm__ volatile(
1720         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1721         "mov                    %0, %%"REG_a"       \n\t"
1722         "1:                                         \n\t"
1723         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1724         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1725         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1726         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1727         "pand                %%mm4, %%mm0           \n\t"
1728         "pand                %%mm4, %%mm1           \n\t"
1729         "pand                %%mm4, %%mm2           \n\t"
1730         "pand                %%mm4, %%mm3           \n\t"
1731         "packuswb            %%mm1, %%mm0           \n\t"
1732         "packuswb            %%mm3, %%mm2           \n\t"
1733         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1734         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1735         "add                    $8, %%"REG_a"       \n\t"
1736         " js                    1b                  \n\t"
1737         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1738         : "%"REG_a
1739     );
1740 }
1741
1742 static av_always_inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1743                                               const uint8_t *src, int width)
1744 {
1745     __asm__ volatile(
1746         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1747         "mov                    %0, %%"REG_a"       \n\t"
1748         "1:                                         \n\t"
1749         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1750         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1751         "movq                %%mm0, %%mm2           \n\t"
1752         "movq                %%mm1, %%mm3           \n\t"
1753         "pand                %%mm4, %%mm0           \n\t"
1754         "pand                %%mm4, %%mm1           \n\t"
1755         "psrlw                  $8, %%mm2           \n\t"
1756         "psrlw                  $8, %%mm3           \n\t"
1757         "packuswb            %%mm1, %%mm0           \n\t"
1758         "packuswb            %%mm3, %%mm2           \n\t"
1759         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1760         "movq                %%mm2, (%3, %%"REG_a") \n\t"
1761         "add                    $8, %%"REG_a"       \n\t"
1762         " js                    1b                  \n\t"
1763         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1764         : "%"REG_a
1765     );
1766 }
1767
1768 static void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1769                              const uint8_t *src1, const uint8_t *src2,
1770                              int width, uint32_t *unused)
1771 {
1772     RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1773 }
1774
1775 static void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1776                              const uint8_t *src1, const uint8_t *src2,
1777                              int width, uint32_t *unused)
1778 {
1779     RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1780 }
1781 #endif /* !COMPILE_TEMPLATE_MMX2 */
1782
1783 static av_always_inline void RENAME(bgr24ToY_mmx)(int16_t *dst, const uint8_t *src,
1784                                                   int width, enum PixelFormat srcFormat)
1785 {
1786
1787     if(srcFormat == PIX_FMT_BGR24) {
1788         __asm__ volatile(
1789             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1790             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1791             :
1792         );
1793     } else {
1794         __asm__ volatile(
1795             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1796             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1797             :
1798         );
1799     }
1800
1801     __asm__ volatile(
1802         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1803         "mov                        %2, %%"REG_a"   \n\t"
1804         "pxor                    %%mm7, %%mm7       \n\t"
1805         "1:                                         \n\t"
1806         PREFETCH"               64(%0)              \n\t"
1807         "movd                     (%0), %%mm0       \n\t"
1808         "movd                    2(%0), %%mm1       \n\t"
1809         "movd                    6(%0), %%mm2       \n\t"
1810         "movd                    8(%0), %%mm3       \n\t"
1811         "add                       $12, %0          \n\t"
1812         "punpcklbw               %%mm7, %%mm0       \n\t"
1813         "punpcklbw               %%mm7, %%mm1       \n\t"
1814         "punpcklbw               %%mm7, %%mm2       \n\t"
1815         "punpcklbw               %%mm7, %%mm3       \n\t"
1816         "pmaddwd                 %%mm5, %%mm0       \n\t"
1817         "pmaddwd                 %%mm6, %%mm1       \n\t"
1818         "pmaddwd                 %%mm5, %%mm2       \n\t"
1819         "pmaddwd                 %%mm6, %%mm3       \n\t"
1820         "paddd                   %%mm1, %%mm0       \n\t"
1821         "paddd                   %%mm3, %%mm2       \n\t"
1822         "paddd                   %%mm4, %%mm0       \n\t"
1823         "paddd                   %%mm4, %%mm2       \n\t"
1824         "psrad                     $9, %%mm0       \n\t"
1825         "psrad                     $9, %%mm2       \n\t"
1826         "packssdw                %%mm2, %%mm0       \n\t"
1827         "movq                %%mm0, (%1, %%"REG_a") \n\t"
1828         "add                        $8, %%"REG_a"   \n\t"
1829         " js                        1b              \n\t"
1830     : "+r" (src)
1831     : "r" (dst+width), "g" ((x86_reg)-2*width)
1832     : "%"REG_a
1833     );
1834 }
1835
1836 static void RENAME(bgr24ToY)(int16_t *dst, const uint8_t *src,
1837                              int width, uint32_t *unused)
1838 {
1839     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1840 }
1841
1842 static void RENAME(rgb24ToY)(int16_t *dst, const uint8_t *src,
1843                              int width, uint32_t *unused)
1844 {
1845     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1846 }
1847
1848 static av_always_inline void RENAME(bgr24ToUV_mmx)(int16_t *dstU, int16_t *dstV,
1849                                                    const uint8_t *src, int width,
1850                                                    enum PixelFormat srcFormat)
1851 {
1852     __asm__ volatile(
1853         "movq                    24(%4), %%mm6       \n\t"
1854         "mov                        %3, %%"REG_a"   \n\t"
1855         "pxor                    %%mm7, %%mm7       \n\t"
1856         "1:                                         \n\t"
1857         PREFETCH"               64(%0)              \n\t"
1858         "movd                     (%0), %%mm0       \n\t"
1859         "movd                    2(%0), %%mm1       \n\t"
1860         "punpcklbw               %%mm7, %%mm0       \n\t"
1861         "punpcklbw               %%mm7, %%mm1       \n\t"
1862         "movq                    %%mm0, %%mm2       \n\t"
1863         "movq                    %%mm1, %%mm3       \n\t"
1864         "pmaddwd                  (%4), %%mm0       \n\t"
1865         "pmaddwd                 8(%4), %%mm1       \n\t"
1866         "pmaddwd                16(%4), %%mm2       \n\t"
1867         "pmaddwd                 %%mm6, %%mm3       \n\t"
1868         "paddd                   %%mm1, %%mm0       \n\t"
1869         "paddd                   %%mm3, %%mm2       \n\t"
1870
1871         "movd                    6(%0), %%mm1       \n\t"
1872         "movd                    8(%0), %%mm3       \n\t"
1873         "add                       $12, %0          \n\t"
1874         "punpcklbw               %%mm7, %%mm1       \n\t"
1875         "punpcklbw               %%mm7, %%mm3       \n\t"
1876         "movq                    %%mm1, %%mm4       \n\t"
1877         "movq                    %%mm3, %%mm5       \n\t"
1878         "pmaddwd                  (%4), %%mm1       \n\t"
1879         "pmaddwd                 8(%4), %%mm3       \n\t"
1880         "pmaddwd                16(%4), %%mm4       \n\t"
1881         "pmaddwd                 %%mm6, %%mm5       \n\t"
1882         "paddd                   %%mm3, %%mm1       \n\t"
1883         "paddd                   %%mm5, %%mm4       \n\t"
1884
1885         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1886         "paddd                   %%mm3, %%mm0       \n\t"
1887         "paddd                   %%mm3, %%mm2       \n\t"
1888         "paddd                   %%mm3, %%mm1       \n\t"
1889         "paddd                   %%mm3, %%mm4       \n\t"
1890         "psrad                     $9, %%mm0       \n\t"
1891         "psrad                     $9, %%mm2       \n\t"
1892         "psrad                     $9, %%mm1       \n\t"
1893         "psrad                     $9, %%mm4       \n\t"
1894         "packssdw                %%mm1, %%mm0       \n\t"
1895         "packssdw                %%mm4, %%mm2       \n\t"
1896         "movq                %%mm0, (%1, %%"REG_a") \n\t"
1897         "movq                %%mm2, (%2, %%"REG_a") \n\t"
1898         "add                        $8, %%"REG_a"   \n\t"
1899         " js                        1b              \n\t"
1900     : "+r" (src)
1901     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-2*width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1902     : "%"REG_a
1903     );
1904 }
1905
1906 static void RENAME(bgr24ToUV)(int16_t *dstU, int16_t *dstV,
1907                               const uint8_t *src1, const uint8_t *src2,
1908                               int width, uint32_t *unused)
1909 {
1910     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1911     assert(src1 == src2);
1912 }
1913
1914 static void RENAME(rgb24ToUV)(int16_t *dstU, int16_t *dstV,
1915                               const uint8_t *src1, const uint8_t *src2,
1916                               int width, uint32_t *unused)
1917 {
1918     assert(src1==src2);
1919     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1920 }
1921
1922 #if !COMPILE_TEMPLATE_MMX2
1923 // bilinear / bicubic scaling
1924 static void RENAME(hScale)(int16_t *dst, int dstW,
1925                            const uint8_t *src, const int16_t *filter,
1926                            const int16_t *filterPos, int filterSize)
1927 {
1928     assert(filterSize % 4 == 0 && filterSize>0);
1929     if (filterSize==4) { // Always true for upscaling, sometimes for down, too.
1930         x86_reg counter= -2*dstW;
1931         filter-= counter*2;
1932         filterPos-= counter/2;
1933         dst-= counter/2;
1934         __asm__ volatile(
1935 #if defined(PIC)
1936             "push            %%"REG_b"              \n\t"
1937 #endif
1938             "pxor                %%mm7, %%mm7       \n\t"
1939             "push           %%"REG_BP"              \n\t" // we use 7 regs here ...
1940             "mov             %%"REG_a", %%"REG_BP"  \n\t"
1941             ".p2align                4              \n\t"
1942             "1:                                     \n\t"
1943             "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"
1944             "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"
1945             "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"
1946             "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"
1947             "movd      (%3, %%"REG_a"), %%mm0       \n\t"
1948             "movd      (%3, %%"REG_b"), %%mm2       \n\t"
1949             "punpcklbw           %%mm7, %%mm0       \n\t"
1950             "punpcklbw           %%mm7, %%mm2       \n\t"
1951             "pmaddwd             %%mm1, %%mm0       \n\t"
1952             "pmaddwd             %%mm2, %%mm3       \n\t"
1953             "movq                %%mm0, %%mm4       \n\t"
1954             "punpckldq           %%mm3, %%mm0       \n\t"
1955             "punpckhdq           %%mm3, %%mm4       \n\t"
1956             "paddd               %%mm4, %%mm0       \n\t"
1957             "psrad                  $7, %%mm0       \n\t"
1958             "packssdw            %%mm0, %%mm0       \n\t"
1959             "movd                %%mm0, (%4, %%"REG_BP")    \n\t"
1960             "add                    $4, %%"REG_BP"  \n\t"
1961             " jnc                   1b              \n\t"
1962
1963             "pop            %%"REG_BP"              \n\t"
1964 #if defined(PIC)
1965             "pop             %%"REG_b"              \n\t"
1966 #endif
1967             : "+a" (counter)
1968             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1969 #if !defined(PIC)
1970             : "%"REG_b
1971 #endif
1972         );
1973     } else if (filterSize==8) {
1974         x86_reg counter= -2*dstW;
1975         filter-= counter*4;
1976         filterPos-= counter/2;
1977         dst-= counter/2;
1978         __asm__ volatile(
1979 #if defined(PIC)
1980             "push             %%"REG_b"             \n\t"
1981 #endif
1982             "pxor                 %%mm7, %%mm7      \n\t"
1983             "push            %%"REG_BP"             \n\t" // we use 7 regs here ...
1984             "mov              %%"REG_a", %%"REG_BP" \n\t"
1985             ".p2align                 4             \n\t"
1986             "1:                                     \n\t"
1987             "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"
1988             "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"
1989             "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"
1990             "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"
1991             "movd       (%3, %%"REG_a"), %%mm0      \n\t"
1992             "movd       (%3, %%"REG_b"), %%mm2      \n\t"
1993             "punpcklbw            %%mm7, %%mm0      \n\t"
1994             "punpcklbw            %%mm7, %%mm2      \n\t"
1995             "pmaddwd              %%mm1, %%mm0      \n\t"
1996             "pmaddwd              %%mm2, %%mm3      \n\t"
1997
1998             "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"
1999             "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"
2000             "movd      4(%3, %%"REG_a"), %%mm4      \n\t"
2001             "movd      4(%3, %%"REG_b"), %%mm2      \n\t"
2002             "punpcklbw            %%mm7, %%mm4      \n\t"
2003             "punpcklbw            %%mm7, %%mm2      \n\t"
2004             "pmaddwd              %%mm1, %%mm4      \n\t"
2005             "pmaddwd              %%mm2, %%mm5      \n\t"
2006             "paddd                %%mm4, %%mm0      \n\t"
2007             "paddd                %%mm5, %%mm3      \n\t"
2008             "movq                 %%mm0, %%mm4      \n\t"
2009             "punpckldq            %%mm3, %%mm0      \n\t"
2010             "punpckhdq            %%mm3, %%mm4      \n\t"
2011             "paddd                %%mm4, %%mm0      \n\t"
2012             "psrad                   $7, %%mm0      \n\t"
2013             "packssdw             %%mm0, %%mm0      \n\t"
2014             "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"
2015             "add                     $4, %%"REG_BP" \n\t"
2016             " jnc                    1b             \n\t"
2017
2018             "pop             %%"REG_BP"             \n\t"
2019 #if defined(PIC)
2020             "pop              %%"REG_b"             \n\t"
2021 #endif
2022             : "+a" (counter)
2023             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
2024 #if !defined(PIC)
2025             : "%"REG_b
2026 #endif
2027         );
2028     } else {
2029         const uint8_t *offset = src+filterSize;
2030         x86_reg counter= -2*dstW;
2031         //filter-= counter*filterSize/2;
2032         filterPos-= counter/2;
2033         dst-= counter/2;
2034         __asm__ volatile(
2035             "pxor                  %%mm7, %%mm7     \n\t"
2036             ".p2align                  4            \n\t"
2037             "1:                                     \n\t"
2038             "mov                      %2, %%"REG_c" \n\t"
2039             "movzwl      (%%"REG_c", %0), %%eax     \n\t"
2040             "movzwl     2(%%"REG_c", %0), %%edx     \n\t"
2041             "mov                      %5, %%"REG_c" \n\t"
2042             "pxor                  %%mm4, %%mm4     \n\t"
2043             "pxor                  %%mm5, %%mm5     \n\t"
2044             "2:                                     \n\t"
2045             "movq                   (%1), %%mm1     \n\t"
2046             "movq               (%1, %6), %%mm3     \n\t"
2047             "movd (%%"REG_c", %%"REG_a"), %%mm0     \n\t"
2048             "movd (%%"REG_c", %%"REG_d"), %%mm2     \n\t"
2049             "punpcklbw             %%mm7, %%mm0     \n\t"
2050             "punpcklbw             %%mm7, %%mm2     \n\t"
2051             "pmaddwd               %%mm1, %%mm0     \n\t"
2052             "pmaddwd               %%mm2, %%mm3     \n\t"
2053             "paddd                 %%mm3, %%mm5     \n\t"
2054             "paddd                 %%mm0, %%mm4     \n\t"
2055             "add                      $8, %1        \n\t"
2056             "add                      $4, %%"REG_c" \n\t"
2057             "cmp                      %4, %%"REG_c" \n\t"
2058             " jb                      2b            \n\t"
2059             "add                      %6, %1        \n\t"
2060             "movq                  %%mm4, %%mm0     \n\t"
2061             "punpckldq             %%mm5, %%mm4     \n\t"
2062             "punpckhdq             %%mm5, %%mm0     \n\t"
2063             "paddd                 %%mm0, %%mm4     \n\t"
2064             "psrad                    $7, %%mm4     \n\t"
2065             "packssdw              %%mm4, %%mm4     \n\t"
2066             "mov                      %3, %%"REG_a" \n\t"
2067             "movd                  %%mm4, (%%"REG_a", %0)   \n\t"
2068             "add                      $4, %0        \n\t"
2069             " jnc                     1b            \n\t"
2070
2071             : "+r" (counter), "+r" (filter)
2072             : "m" (filterPos), "m" (dst), "m"(offset),
2073             "m" (src), "r" ((x86_reg)filterSize*2)
2074             : "%"REG_a, "%"REG_c, "%"REG_d
2075         );
2076     }
2077 }
2078 #endif /* !COMPILE_TEMPLATE_MMX2 */
2079
2080 static inline void RENAME(hScale16)(int16_t *dst, int dstW, const uint16_t *src, int srcW, int xInc,
2081                                     const int16_t *filter, const int16_t *filterPos, long filterSize, int shift)
2082 {
2083     int i, j;
2084
2085     assert(filterSize % 4 == 0 && filterSize>0);
2086     if (filterSize==4 && shift<15) { // Always true for upscaling, sometimes for down, too.
2087         x86_reg counter= -2*dstW;
2088         filter-= counter*2;
2089         filterPos-= counter/2;
2090         dst-= counter/2;
2091         __asm__ volatile(
2092             "movd                   %5, %%mm7       \n\t"
2093 #if defined(PIC)
2094             "push            %%"REG_b"              \n\t"
2095 #endif
2096             "push           %%"REG_BP"              \n\t" // we use 7 regs here ...
2097             "mov             %%"REG_a", %%"REG_BP"  \n\t"
2098             ".p2align                4              \n\t"
2099             "1:                                     \n\t"
2100             "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"
2101             "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"
2102             "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"
2103             "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"
2104             "movq      (%3, %%"REG_a", 2), %%mm0    \n\t"
2105             "movq      (%3, %%"REG_b", 2), %%mm2    \n\t"
2106             "pmaddwd             %%mm1, %%mm0       \n\t"
2107             "pmaddwd             %%mm2, %%mm3       \n\t"
2108             "movq                %%mm0, %%mm4       \n\t"
2109             "punpckldq           %%mm3, %%mm0       \n\t"
2110             "punpckhdq           %%mm3, %%mm4       \n\t"
2111             "paddd               %%mm4, %%mm0       \n\t"
2112             "psrad               %%mm7, %%mm0       \n\t"
2113             "packssdw            %%mm0, %%mm0       \n\t"
2114             "movd                %%mm0, (%4, %%"REG_BP")    \n\t"
2115             "add                    $4, %%"REG_BP"  \n\t"
2116             " jnc                   1b              \n\t"
2117
2118             "pop            %%"REG_BP"              \n\t"
2119 #if defined(PIC)
2120             "pop             %%"REG_b"              \n\t"
2121 #endif
2122             : "+a" (counter)
2123             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst), "m"(shift)
2124 #if !defined(PIC)
2125             : "%"REG_b
2126 #endif
2127         );
2128     } else if (filterSize==8 && shift<15) {
2129         x86_reg counter= -2*dstW;
2130         filter-= counter*4;
2131         filterPos-= counter/2;
2132         dst-= counter/2;
2133         __asm__ volatile(
2134             "movd                   %5, %%mm7       \n\t"
2135 #if defined(PIC)
2136             "push            %%"REG_b"              \n\t"
2137 #endif
2138             "push            %%"REG_BP"             \n\t" // we use 7 regs here ...
2139             "mov              %%"REG_a", %%"REG_BP" \n\t"
2140             ".p2align                 4             \n\t"
2141             "1:                                     \n\t"
2142             "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"
2143             "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"
2144             "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"
2145             "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"
2146             "movq       (%3, %%"REG_a", 2), %%mm0   \n\t"
2147             "movq       (%3, %%"REG_b", 2), %%mm2   \n\t"
2148             "pmaddwd              %%mm1, %%mm0      \n\t"
2149             "pmaddwd              %%mm2, %%mm3      \n\t"
2150
2151             "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"
2152             "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"
2153             "movq      8(%3, %%"REG_a", 2), %%mm4   \n\t"
2154             "movq      8(%3, %%"REG_b", 2), %%mm2   \n\t"
2155             "pmaddwd              %%mm1, %%mm4      \n\t"
2156             "pmaddwd              %%mm2, %%mm5      \n\t"
2157             "paddd                %%mm4, %%mm0      \n\t"
2158             "paddd                %%mm5, %%mm3      \n\t"
2159             "movq                 %%mm0, %%mm4      \n\t"
2160             "punpckldq            %%mm3, %%mm0      \n\t"
2161             "punpckhdq            %%mm3, %%mm4      \n\t"
2162             "paddd                %%mm4, %%mm0      \n\t"
2163             "psrad                %%mm7, %%mm0      \n\t"
2164             "packssdw             %%mm0, %%mm0      \n\t"
2165             "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"
2166             "add                     $4, %%"REG_BP" \n\t"
2167             " jnc                    1b             \n\t"
2168
2169             "pop             %%"REG_BP"             \n\t"
2170 #if defined(PIC)
2171             "pop             %%"REG_b"              \n\t"
2172 #endif
2173             : "+a" (counter)
2174             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst), "m"(shift)
2175 #if !defined(PIC)
2176             : "%"REG_b
2177 #endif
2178         );
2179     } else if (shift<15){
2180         const uint16_t *offset = src+filterSize;
2181         x86_reg counter= -2*dstW;
2182         //filter-= counter*filterSize/2;
2183         filterPos-= counter/2;
2184         dst-= counter/2;
2185         __asm__ volatile(
2186             "movd                   %7, %%mm7       \n\t"
2187             ".p2align                  4            \n\t"
2188             "1:                                     \n\t"
2189             "mov                      %2, %%"REG_c" \n\t"
2190             "movzwl      (%%"REG_c", %0), %%eax     \n\t"
2191             "movzwl     2(%%"REG_c", %0), %%edx     \n\t"
2192             "mov                      %5, %%"REG_c" \n\t"
2193             "pxor                  %%mm4, %%mm4     \n\t"
2194             "pxor                  %%mm5, %%mm5     \n\t"
2195             "2:                                     \n\t"
2196             "movq                   (%1), %%mm1     \n\t"
2197             "movq               (%1, %6), %%mm3     \n\t"
2198             "movq (%%"REG_c", %%"REG_a", 2), %%mm0     \n\t"
2199             "movq (%%"REG_c", %%"REG_d", 2), %%mm2     \n\t"
2200             "pmaddwd               %%mm1, %%mm0     \n\t"
2201             "pmaddwd               %%mm2, %%mm3     \n\t"
2202             "paddd                 %%mm3, %%mm5     \n\t"
2203             "paddd                 %%mm0, %%mm4     \n\t"
2204             "add                      $8, %1        \n\t"
2205             "add                      $8, %%"REG_c" \n\t"
2206             "cmp                      %4, %%"REG_c" \n\t"
2207             " jb                      2b            \n\t"
2208             "add                      %6, %1        \n\t"
2209             "movq                  %%mm4, %%mm0     \n\t"
2210             "punpckldq             %%mm5, %%mm4     \n\t"
2211             "punpckhdq             %%mm5, %%mm0     \n\t"
2212             "paddd                 %%mm0, %%mm4     \n\t"
2213             "psrad                 %%mm7, %%mm4     \n\t"
2214             "packssdw              %%mm4, %%mm4     \n\t"
2215             "mov                      %3, %%"REG_a" \n\t"
2216             "movd                  %%mm4, (%%"REG_a", %0)   \n\t"
2217             "add                      $4, %0        \n\t"
2218             " jnc                     1b            \n\t"
2219
2220             : "+r" (counter), "+r" (filter)
2221             : "m" (filterPos), "m" (dst), "m"(offset),
2222             "m" (src), "r" ((x86_reg)filterSize*2), "m"(shift)
2223             : "%"REG_a, "%"REG_c, "%"REG_d
2224         );
2225     } else
2226     for (i=0; i<dstW; i++) {
2227         int srcPos= filterPos[i];
2228         int val=0;
2229         for (j=0; j<filterSize; j++) {
2230             val += ((int)src[srcPos + j])*filter[filterSize*i + j];
2231         }
2232         dst[i] = FFMIN(val>>shift, (1<<15)-1); // the cubic equation does overflow ...
2233     }
2234 }
2235
2236
2237 #if COMPILE_TEMPLATE_MMX2
2238 static void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
2239                                  int dstWidth, const uint8_t *src,
2240                                  int srcW, int xInc)
2241 {
2242     int16_t *filterPos = c->hLumFilterPos;
2243     int16_t *filter    = c->hLumFilter;
2244     void    *mmx2FilterCode= c->lumMmx2FilterCode;
2245     int i;
2246 #if defined(PIC)
2247     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2248 #endif
2249
2250     __asm__ volatile(
2251 #if defined(PIC)
2252         "mov               %%"REG_b", %5        \n\t"
2253 #endif
2254         "pxor                  %%mm7, %%mm7     \n\t"
2255         "mov                      %0, %%"REG_c" \n\t"
2256         "mov                      %1, %%"REG_D" \n\t"
2257         "mov                      %2, %%"REG_d" \n\t"
2258         "mov                      %3, %%"REG_b" \n\t"
2259         "xor               %%"REG_a", %%"REG_a" \n\t" // i
2260         PREFETCH"        (%%"REG_c")            \n\t"
2261         PREFETCH"      32(%%"REG_c")            \n\t"
2262         PREFETCH"      64(%%"REG_c")            \n\t"
2263
2264 #if ARCH_X86_64
2265 #define CALL_MMX2_FILTER_CODE \
2266         "movl            (%%"REG_b"), %%esi     \n\t"\
2267         "call                    *%4            \n\t"\
2268         "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
2269         "add               %%"REG_S", %%"REG_c" \n\t"\
2270         "add               %%"REG_a", %%"REG_D" \n\t"\
2271         "xor               %%"REG_a", %%"REG_a" \n\t"\
2272
2273 #else
2274 #define CALL_MMX2_FILTER_CODE \
2275         "movl (%%"REG_b"), %%esi        \n\t"\
2276         "call         *%4                       \n\t"\
2277         "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
2278         "add               %%"REG_a", %%"REG_D" \n\t"\
2279         "xor               %%"REG_a", %%"REG_a" \n\t"\
2280
2281 #endif /* ARCH_X86_64 */
2282
2283         CALL_MMX2_FILTER_CODE
2284         CALL_MMX2_FILTER_CODE
2285         CALL_MMX2_FILTER_CODE
2286         CALL_MMX2_FILTER_CODE
2287         CALL_MMX2_FILTER_CODE
2288         CALL_MMX2_FILTER_CODE
2289         CALL_MMX2_FILTER_CODE
2290         CALL_MMX2_FILTER_CODE
2291
2292 #if defined(PIC)
2293         "mov                      %5, %%"REG_b" \n\t"
2294 #endif
2295         :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
2296            "m" (mmx2FilterCode)
2297 #if defined(PIC)
2298           ,"m" (ebxsave)
2299 #endif
2300         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2301 #if !defined(PIC)
2302          ,"%"REG_b
2303 #endif
2304     );
2305
2306     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
2307         dst[i] = src[srcW-1]*128;
2308 }
2309
2310 static void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2,
2311                                  int dstWidth, const uint8_t *src1,
2312                                  const uint8_t *src2, int srcW, int xInc)
2313 {
2314     int16_t *filterPos = c->hChrFilterPos;
2315     int16_t *filter    = c->hChrFilter;
2316     void    *mmx2FilterCode= c->chrMmx2FilterCode;
2317     int i;
2318 #if defined(PIC)
2319     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2320 #endif
2321
2322     __asm__ volatile(
2323 #if defined(PIC)
2324         "mov          %%"REG_b", %7         \n\t"
2325 #endif
2326         "pxor             %%mm7, %%mm7      \n\t"
2327         "mov                 %0, %%"REG_c"  \n\t"
2328         "mov                 %1, %%"REG_D"  \n\t"
2329         "mov                 %2, %%"REG_d"  \n\t"
2330         "mov                 %3, %%"REG_b"  \n\t"
2331         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2332         PREFETCH"   (%%"REG_c")             \n\t"
2333         PREFETCH" 32(%%"REG_c")             \n\t"
2334         PREFETCH" 64(%%"REG_c")             \n\t"
2335
2336         CALL_MMX2_FILTER_CODE
2337         CALL_MMX2_FILTER_CODE
2338         CALL_MMX2_FILTER_CODE
2339         CALL_MMX2_FILTER_CODE
2340         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2341         "mov                 %5, %%"REG_c"  \n\t" // src
2342         "mov                 %6, %%"REG_D"  \n\t" // buf2
2343         PREFETCH"   (%%"REG_c")             \n\t"
2344         PREFETCH" 32(%%"REG_c")             \n\t"
2345         PREFETCH" 64(%%"REG_c")             \n\t"
2346
2347         CALL_MMX2_FILTER_CODE
2348         CALL_MMX2_FILTER_CODE
2349         CALL_MMX2_FILTER_CODE
2350         CALL_MMX2_FILTER_CODE
2351
2352 #if defined(PIC)
2353         "mov %7, %%"REG_b"    \n\t"
2354 #endif
2355         :: "m" (src1), "m" (dst1), "m" (filter), "m" (filterPos),
2356            "m" (mmx2FilterCode), "m" (src2), "m"(dst2)
2357 #if defined(PIC)
2358           ,"m" (ebxsave)
2359 #endif
2360         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2361 #if !defined(PIC)
2362          ,"%"REG_b
2363 #endif
2364     );
2365
2366     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
2367         dst1[i] = src1[srcW-1]*128;
2368         dst2[i] = src2[srcW-1]*128;
2369     }
2370 }
2371 #endif /* COMPILE_TEMPLATE_MMX2 */
2372
2373 static av_cold void RENAME(sws_init_swScale)(SwsContext *c)
2374 {
2375     enum PixelFormat srcFormat = c->srcFormat,
2376                      dstFormat = c->dstFormat;
2377
2378     if (!is16BPS(dstFormat) && !is9_OR_10BPS(dstFormat) && dstFormat != PIX_FMT_NV12
2379         && dstFormat != PIX_FMT_NV21 && !(c->flags & SWS_BITEXACT)) {
2380             if (c->flags & SWS_ACCURATE_RND) {
2381                 c->yuv2yuv1 = RENAME(yuv2yuv1_ar    );
2382                 c->yuv2yuvX = RENAME(yuv2yuvX_ar    );
2383                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
2384                     switch (c->dstFormat) {
2385                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;
2386                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;
2387                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;
2388                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;
2389                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;
2390                     default: break;
2391                     }
2392                 }
2393             } else {
2394                 int should_dither= isNBPS(c->srcFormat) || is16BPS(c->srcFormat);
2395                 c->yuv2yuv1 = should_dither ? RENAME(yuv2yuv1_ar    ) : RENAME(yuv2yuv1    );
2396                 c->yuv2yuvX = RENAME(yuv2yuvX    );
2397                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
2398                     switch (c->dstFormat) {
2399                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;
2400                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;
2401                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;
2402                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;
2403                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;
2404                     default: break;
2405                     }
2406                 }
2407             }
2408         if (!(c->flags & SWS_FULL_CHR_H_INT)) {
2409             switch (c->dstFormat) {
2410             case PIX_FMT_RGB32:
2411                 c->yuv2packed1 = RENAME(yuv2rgb32_1);
2412                 c->yuv2packed2 = RENAME(yuv2rgb32_2);
2413                 break;
2414             case PIX_FMT_BGR24:
2415                 c->yuv2packed1 = RENAME(yuv2bgr24_1);
2416                 c->yuv2packed2 = RENAME(yuv2bgr24_2);
2417                 break;
2418             case PIX_FMT_RGB555:
2419                 c->yuv2packed1 = RENAME(yuv2rgb555_1);
2420                 c->yuv2packed2 = RENAME(yuv2rgb555_2);
2421                 break;
2422             case PIX_FMT_RGB565:
2423                 c->yuv2packed1 = RENAME(yuv2rgb565_1);
2424                 c->yuv2packed2 = RENAME(yuv2rgb565_2);
2425                 break;
2426             case PIX_FMT_YUYV422:
2427                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);
2428                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);
2429                 break;
2430             default:
2431                 break;
2432             }
2433         }
2434     }
2435
2436 #if !COMPILE_TEMPLATE_MMX2
2437     c->hScale       = RENAME(hScale      );
2438 #endif /* !COMPILE_TEMPLATE_MMX2 */
2439
2440     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2441 #if COMPILE_TEMPLATE_MMX2
2442     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
2443     {
2444         c->hyscale_fast = RENAME(hyscale_fast);
2445         c->hcscale_fast = RENAME(hcscale_fast);
2446     } else {
2447 #endif /* COMPILE_TEMPLATE_MMX2 */
2448         c->hyscale_fast = NULL;
2449         c->hcscale_fast = NULL;
2450 #if COMPILE_TEMPLATE_MMX2
2451     }
2452 #endif /* COMPILE_TEMPLATE_MMX2 */
2453
2454 #if !COMPILE_TEMPLATE_MMX2
2455     switch(srcFormat) {
2456         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;
2457         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;
2458         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;
2459         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;
2460         case PIX_FMT_GRAY16LE :
2461         case PIX_FMT_YUV420P9LE:
2462         case PIX_FMT_YUV422P10LE:
2463         case PIX_FMT_YUV420P10LE:
2464         case PIX_FMT_YUV420P16LE:
2465         case PIX_FMT_YUV422P16LE:
2466         case PIX_FMT_YUV444P16LE: c->hScale16= RENAME(hScale16); break;
2467     }
2468 #endif /* !COMPILE_TEMPLATE_MMX2 */
2469     if (!c->chrSrcHSubSample) {
2470         switch(srcFormat) {
2471         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
2472         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
2473         default: break;
2474         }
2475     }
2476
2477     switch (srcFormat) {
2478 #if !COMPILE_TEMPLATE_MMX2
2479     case PIX_FMT_YUYV422  :
2480     case PIX_FMT_Y400A    :
2481                             c->lumToYV12 = RENAME(yuy2ToY); break;
2482     case PIX_FMT_UYVY422  :
2483                             c->lumToYV12 = RENAME(uyvyToY); break;
2484 #endif /* !COMPILE_TEMPLATE_MMX2 */
2485     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
2486     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
2487     default: break;
2488     }
2489 #if !COMPILE_TEMPLATE_MMX2
2490     if (c->alpPixBuf) {
2491         switch (srcFormat) {
2492         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;
2493         default: break;
2494         }
2495     }
2496 #endif /* !COMPILE_TEMPLATE_MMX2 */
2497     if(isAnyRGB(c->srcFormat))
2498         c->hScale16= RENAME(hScale16);
2499 }