Merge remote-tracking branch 'qatar/master'
[ffmpeg.git] / libswscale / x86 / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #undef REAL_MOVNTQ
22 #undef MOVNTQ
23 #undef PREFETCH
24
25 #if COMPILE_TEMPLATE_MMX2
26 #define PREFETCH "prefetchnta"
27 #else
28 #define PREFETCH  " # nop"
29 #endif
30
31 #if COMPILE_TEMPLATE_MMX2
32 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
33 #else
34 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
35 #endif
36 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
37
38 #if !COMPILE_TEMPLATE_MMX2
39 static av_always_inline void
40 dither_8to16(SwsContext *c, const uint8_t *srcDither, int rot)
41 {
42     if (rot) {
43         __asm__ volatile("pxor      %%mm0, %%mm0\n\t"
44                          "movq       (%0), %%mm3\n\t"
45                          "movq      %%mm3, %%mm4\n\t"
46                          "psrlq       $24, %%mm3\n\t"
47                          "psllq       $40, %%mm4\n\t"
48                          "por       %%mm4, %%mm3\n\t"
49                          "movq      %%mm3, %%mm4\n\t"
50                          "punpcklbw %%mm0, %%mm3\n\t"
51                          "punpckhbw %%mm0, %%mm4\n\t"
52                          "psraw        $4, %%mm3\n\t"
53                          "psraw        $4, %%mm4\n\t"
54                          "movq      %%mm3, "DITHER16"+0(%1)\n\t"
55                          "movq      %%mm4, "DITHER16"+8(%1)\n\t"
56                          :: "r"(srcDither), "r"(&c->redDither)
57                          );
58     } else {
59         __asm__ volatile("pxor      %%mm0, %%mm0\n\t"
60                          "movq       (%0), %%mm3\n\t"
61                          "movq      %%mm3, %%mm4\n\t"
62                          "punpcklbw %%mm0, %%mm3\n\t"
63                          "punpckhbw %%mm0, %%mm4\n\t"
64                          "psraw        $4, %%mm3\n\t"
65                          "psraw        $4, %%mm4\n\t"
66                          "movq      %%mm3, "DITHER16"+0(%1)\n\t"
67                          "movq      %%mm4, "DITHER16"+8(%1)\n\t"
68                          :: "r"(srcDither), "r"(&c->redDither)
69                          );
70     }
71 }
72 #endif
73
74 static void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc,
75                              const int16_t *chrUSrc, const int16_t *chrVSrc,
76                              const int16_t *alpSrc,
77                              uint8_t *dst[4], int dstW, int chrDstW)
78 {
79     int p= 4;
80     const int16_t *src[4]= {
81         lumSrc + dstW,     chrUSrc + chrDstW,
82         chrVSrc + chrDstW, alpSrc + dstW
83     };
84     x86_reg counter[4]= { dstW, chrDstW, chrDstW, dstW };
85
86     while (p--) {
87         if (dst[p]) {
88             __asm__ volatile(
89                 "mov %2, %%"REG_a"                    \n\t"
90                 ".p2align               4             \n\t" /* FIXME Unroll? */
91                 "1:                                   \n\t"
92                 "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"
93                 "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"
94                 "psraw                 $7, %%mm0      \n\t"
95                 "psraw                 $7, %%mm1      \n\t"
96                 "packuswb           %%mm1, %%mm0      \n\t"
97                 MOVNTQ(%%mm0, (%1, %%REGa))
98                 "add                   $8, %%"REG_a"  \n\t"
99                 "jnc                   1b             \n\t"
100                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
101                    "g" (-counter[p])
102                 : "%"REG_a
103             );
104         }
105     }
106 }
107
108 static void RENAME(yuv2yuv1_ar)(SwsContext *c, const int16_t *lumSrc,
109                                 const int16_t *chrUSrc, const int16_t *chrVSrc,
110                                 const int16_t *alpSrc,
111                                 uint8_t *dst[4], int dstW, int chrDstW)
112 {
113     int p= 4;
114     const int16_t *src[4]= {
115         lumSrc + dstW,     chrUSrc + chrDstW,
116         chrVSrc + chrDstW, alpSrc + dstW
117     };
118     x86_reg counter[4]= { dstW, chrDstW, chrDstW, dstW };
119     const uint8_t *lumDither = c->lumDither8, *chrDither = c->chrDither8;
120
121     while (p--) {
122         if (dst[p]) {
123             int i;
124             for(i=0; i<8; i++) c->dither16[i] = (p == 2 || p == 3) ? lumDither[i] : chrDither[i];
125             __asm__ volatile(
126                 "mov %2, %%"REG_a"                    \n\t"
127                 "movq    "DITHER16"+0(%3), %%mm6      \n\t"
128                 "movq    "DITHER16"+8(%3), %%mm7      \n\t"
129                 ".p2align                4            \n\t" /* FIXME Unroll? */
130                 "1:                                   \n\t"
131                 "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"
132                 "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"
133                 "paddsw             %%mm6, %%mm0      \n\t"
134                 "paddsw             %%mm7, %%mm1      \n\t"
135                 "psraw                 $7, %%mm0      \n\t"
136                 "psraw                 $7, %%mm1      \n\t"
137                 "packuswb           %%mm1, %%mm0      \n\t"
138                 MOVNTQ(%%mm0, (%1, %%REGa))
139                 "add                   $8, %%"REG_a"  \n\t"
140                 "jnc                   1b             \n\t"
141                 :: "r" (src[p]), "r" (dst[p] + counter[p]),
142                    "g" (-counter[p]), "r"(&c->redDither)
143                 : "%"REG_a
144             );
145         }
146     }
147 }
148
149 #define YSCALEYUV2PACKEDX_UV \
150     __asm__ volatile(\
151         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
152         ".p2align                      4                \n\t"\
153         "nop                                            \n\t"\
154         "1:                                             \n\t"\
155         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
156         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
157         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
158         "movq                      %%mm3, %%mm4         \n\t"\
159         ".p2align                      4                \n\t"\
160         "2:                                             \n\t"\
161         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
162         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
163         "add                          %6, %%"REG_S"     \n\t" \
164         "movq     (%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
165         "add                         $16, %%"REG_d"     \n\t"\
166         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
167         "pmulhw                    %%mm0, %%mm2         \n\t"\
168         "pmulhw                    %%mm0, %%mm5         \n\t"\
169         "paddw                     %%mm2, %%mm3         \n\t"\
170         "paddw                     %%mm5, %%mm4         \n\t"\
171         "test                  %%"REG_S", %%"REG_S"     \n\t"\
172         " jnz                         2b                \n\t"\
173
174 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
175     "lea                "offset"(%0), %%"REG_d"     \n\t"\
176     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
177     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
178     "movq                    "#dst1", "#dst2"       \n\t"\
179     ".p2align                      4                \n\t"\
180     "2:                                             \n\t"\
181     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
182     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
183     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
184     "add                         $16, %%"REG_d"            \n\t"\
185     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
186     "pmulhw                 "#coeff", "#src1"       \n\t"\
187     "pmulhw                 "#coeff", "#src2"       \n\t"\
188     "paddw                   "#src1", "#dst1"       \n\t"\
189     "paddw                   "#src2", "#dst2"       \n\t"\
190     "test                  %%"REG_S", %%"REG_S"     \n\t"\
191     " jnz                         2b                \n\t"\
192
193 #define YSCALEYUV2PACKEDX \
194     YSCALEYUV2PACKEDX_UV \
195     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
196
197 #define YSCALEYUV2PACKEDX_END                     \
198         :: "r" (&c->redDither),                   \
199             "m" (dummy), "m" (dummy), "m" (dummy),\
200             "r" (dest), "m" (dstW_reg), "m"(uv_off) \
201         : "%"REG_a, "%"REG_d, "%"REG_S            \
202     );
203
204 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
205     __asm__ volatile(\
206         "xor %%"REG_a", %%"REG_a"                       \n\t"\
207         ".p2align                      4                \n\t"\
208         "nop                                            \n\t"\
209         "1:                                             \n\t"\
210         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
211         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
212         "pxor                      %%mm4, %%mm4         \n\t"\
213         "pxor                      %%mm5, %%mm5         \n\t"\
214         "pxor                      %%mm6, %%mm6         \n\t"\
215         "pxor                      %%mm7, %%mm7         \n\t"\
216         ".p2align                      4                \n\t"\
217         "2:                                             \n\t"\
218         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
219         "add                          %6, %%"REG_S"      \n\t" \
220         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
221         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
222         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
223         "movq                      %%mm0, %%mm3         \n\t"\
224         "punpcklwd                 %%mm1, %%mm0         \n\t"\
225         "punpckhwd                 %%mm1, %%mm3         \n\t"\
226         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
227         "pmaddwd                   %%mm1, %%mm0         \n\t"\
228         "pmaddwd                   %%mm1, %%mm3         \n\t"\
229         "paddd                     %%mm0, %%mm4         \n\t"\
230         "paddd                     %%mm3, %%mm5         \n\t"\
231         "add                          %6, %%"REG_S"      \n\t" \
232         "movq     (%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
233         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
234         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
235         "test                  %%"REG_S", %%"REG_S"     \n\t"\
236         "movq                      %%mm2, %%mm0         \n\t"\
237         "punpcklwd                 %%mm3, %%mm2         \n\t"\
238         "punpckhwd                 %%mm3, %%mm0         \n\t"\
239         "pmaddwd                   %%mm1, %%mm2         \n\t"\
240         "pmaddwd                   %%mm1, %%mm0         \n\t"\
241         "paddd                     %%mm2, %%mm6         \n\t"\
242         "paddd                     %%mm0, %%mm7         \n\t"\
243         " jnz                         2b                \n\t"\
244         "psrad                       $16, %%mm4         \n\t"\
245         "psrad                       $16, %%mm5         \n\t"\
246         "psrad                       $16, %%mm6         \n\t"\
247         "psrad                       $16, %%mm7         \n\t"\
248         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
249         "packssdw                  %%mm5, %%mm4         \n\t"\
250         "packssdw                  %%mm7, %%mm6         \n\t"\
251         "paddw                     %%mm0, %%mm4         \n\t"\
252         "paddw                     %%mm0, %%mm6         \n\t"\
253         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
254         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
255
256 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
257     "lea                "offset"(%0), %%"REG_d"     \n\t"\
258     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
259     "pxor                      %%mm1, %%mm1         \n\t"\
260     "pxor                      %%mm5, %%mm5         \n\t"\
261     "pxor                      %%mm7, %%mm7         \n\t"\
262     "pxor                      %%mm6, %%mm6         \n\t"\
263     ".p2align                      4                \n\t"\
264     "2:                                             \n\t"\
265     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
266     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
267     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
268     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
269     "movq                      %%mm0, %%mm3         \n\t"\
270     "punpcklwd                 %%mm4, %%mm0         \n\t"\
271     "punpckhwd                 %%mm4, %%mm3         \n\t"\
272     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
273     "pmaddwd                   %%mm4, %%mm0         \n\t"\
274     "pmaddwd                   %%mm4, %%mm3         \n\t"\
275     "paddd                     %%mm0, %%mm1         \n\t"\
276     "paddd                     %%mm3, %%mm5         \n\t"\
277     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
278     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
279     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
280     "test                  %%"REG_S", %%"REG_S"     \n\t"\
281     "movq                      %%mm2, %%mm0         \n\t"\
282     "punpcklwd                 %%mm3, %%mm2         \n\t"\
283     "punpckhwd                 %%mm3, %%mm0         \n\t"\
284     "pmaddwd                   %%mm4, %%mm2         \n\t"\
285     "pmaddwd                   %%mm4, %%mm0         \n\t"\
286     "paddd                     %%mm2, %%mm7         \n\t"\
287     "paddd                     %%mm0, %%mm6         \n\t"\
288     " jnz                         2b                \n\t"\
289     "psrad                       $16, %%mm1         \n\t"\
290     "psrad                       $16, %%mm5         \n\t"\
291     "psrad                       $16, %%mm7         \n\t"\
292     "psrad                       $16, %%mm6         \n\t"\
293     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
294     "packssdw                  %%mm5, %%mm1         \n\t"\
295     "packssdw                  %%mm6, %%mm7         \n\t"\
296     "paddw                     %%mm0, %%mm1         \n\t"\
297     "paddw                     %%mm0, %%mm7         \n\t"\
298     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
299     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
300
301 #define YSCALEYUV2PACKEDX_ACCURATE \
302     YSCALEYUV2PACKEDX_ACCURATE_UV \
303     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
304
305 #define YSCALEYUV2RGBX \
306     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
307     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
308     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
309     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
310     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
311     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
312     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
313     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
314     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
315     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
316     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
317     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
318     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
319     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
320     "paddw           %%mm3, %%mm4       \n\t"\
321     "movq            %%mm2, %%mm0       \n\t"\
322     "movq            %%mm5, %%mm6       \n\t"\
323     "movq            %%mm4, %%mm3       \n\t"\
324     "punpcklwd       %%mm2, %%mm2       \n\t"\
325     "punpcklwd       %%mm5, %%mm5       \n\t"\
326     "punpcklwd       %%mm4, %%mm4       \n\t"\
327     "paddw           %%mm1, %%mm2       \n\t"\
328     "paddw           %%mm1, %%mm5       \n\t"\
329     "paddw           %%mm1, %%mm4       \n\t"\
330     "punpckhwd       %%mm0, %%mm0       \n\t"\
331     "punpckhwd       %%mm6, %%mm6       \n\t"\
332     "punpckhwd       %%mm3, %%mm3       \n\t"\
333     "paddw           %%mm7, %%mm0       \n\t"\
334     "paddw           %%mm7, %%mm6       \n\t"\
335     "paddw           %%mm7, %%mm3       \n\t"\
336     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
337     "packuswb        %%mm0, %%mm2       \n\t"\
338     "packuswb        %%mm6, %%mm5       \n\t"\
339     "packuswb        %%mm3, %%mm4       \n\t"\
340
341 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
342     "movq       "#b", "#q2"     \n\t" /* B */\
343     "movq       "#r", "#t"      \n\t" /* R */\
344     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
345     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
346     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
347     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
348     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
349     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
350     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
351     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
352     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
353     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
354 \
355     MOVNTQ(   q0,   (dst, index, 4))\
356     MOVNTQ(    b,  8(dst, index, 4))\
357     MOVNTQ(   q2, 16(dst, index, 4))\
358     MOVNTQ(   q3, 24(dst, index, 4))\
359 \
360     "add      $8, "#index"      \n\t"\
361     "cmp "#dstw", "#index"      \n\t"\
362     " jb      1b                \n\t"
363 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
364
365 static void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,
366                                    const int16_t **lumSrc, int lumFilterSize,
367                                    const int16_t *chrFilter, const int16_t **chrUSrc,
368                                    const int16_t **chrVSrc,
369                                    int chrFilterSize, const int16_t **alpSrc,
370                                    uint8_t *dest, int dstW, int dstY)
371 {
372     x86_reg dummy=0;
373     x86_reg dstW_reg = dstW;
374     x86_reg uv_off = c->uv_offx2;
375
376     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
377         YSCALEYUV2PACKEDX_ACCURATE
378         YSCALEYUV2RGBX
379         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
380         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
381         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
382         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
383         "movq               "Y_TEMP"(%0), %%mm5         \n\t"
384         "psraw                        $3, %%mm1         \n\t"
385         "psraw                        $3, %%mm7         \n\t"
386         "packuswb                  %%mm7, %%mm1         \n\t"
387         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
388         YSCALEYUV2PACKEDX_END
389     } else {
390         YSCALEYUV2PACKEDX_ACCURATE
391         YSCALEYUV2RGBX
392         "pcmpeqd %%mm7, %%mm7 \n\t"
393         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
394         YSCALEYUV2PACKEDX_END
395     }
396 }
397
398 static void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,
399                                 const int16_t **lumSrc, int lumFilterSize,
400                                 const int16_t *chrFilter, const int16_t **chrUSrc,
401                                 const int16_t **chrVSrc,
402                                 int chrFilterSize, const int16_t **alpSrc,
403                                 uint8_t *dest, int dstW, int dstY)
404 {
405     x86_reg dummy=0;
406     x86_reg dstW_reg = dstW;
407     x86_reg uv_off = c->uv_offx2;
408
409     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
410         YSCALEYUV2PACKEDX
411         YSCALEYUV2RGBX
412         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
413         "psraw                        $3, %%mm1         \n\t"
414         "psraw                        $3, %%mm7         \n\t"
415         "packuswb                  %%mm7, %%mm1         \n\t"
416         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
417         YSCALEYUV2PACKEDX_END
418     } else {
419         YSCALEYUV2PACKEDX
420         YSCALEYUV2RGBX
421         "pcmpeqd %%mm7, %%mm7 \n\t"
422         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
423         YSCALEYUV2PACKEDX_END
424     }
425 }
426
427 #define REAL_WRITERGB16(dst, dstw, index) \
428     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
429     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
430     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
431     "psrlq           $3, %%mm2  \n\t"\
432 \
433     "movq         %%mm2, %%mm1  \n\t"\
434     "movq         %%mm4, %%mm3  \n\t"\
435 \
436     "punpcklbw    %%mm7, %%mm3  \n\t"\
437     "punpcklbw    %%mm5, %%mm2  \n\t"\
438     "punpckhbw    %%mm7, %%mm4  \n\t"\
439     "punpckhbw    %%mm5, %%mm1  \n\t"\
440 \
441     "psllq           $3, %%mm3  \n\t"\
442     "psllq           $3, %%mm4  \n\t"\
443 \
444     "por          %%mm3, %%mm2  \n\t"\
445     "por          %%mm4, %%mm1  \n\t"\
446 \
447     MOVNTQ(%%mm2,  (dst, index, 2))\
448     MOVNTQ(%%mm1, 8(dst, index, 2))\
449 \
450     "add             $8, "#index"   \n\t"\
451     "cmp        "#dstw", "#index"   \n\t"\
452     " jb             1b             \n\t"
453 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
454
455 static void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,
456                                     const int16_t **lumSrc, int lumFilterSize,
457                                     const int16_t *chrFilter, const int16_t **chrUSrc,
458                                     const int16_t **chrVSrc,
459                                     int chrFilterSize, const int16_t **alpSrc,
460                                     uint8_t *dest, int dstW, int dstY)
461 {
462     x86_reg dummy=0;
463     x86_reg dstW_reg = dstW;
464     x86_reg uv_off = c->uv_offx2;
465
466     YSCALEYUV2PACKEDX_ACCURATE
467     YSCALEYUV2RGBX
468     "pxor %%mm7, %%mm7 \n\t"
469     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
470 #ifdef DITHER1XBPP
471     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
472     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
473     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
474 #endif
475     WRITERGB16(%4, %5, %%REGa)
476     YSCALEYUV2PACKEDX_END
477 }
478
479 static void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,
480                                  const int16_t **lumSrc, int lumFilterSize,
481                                  const int16_t *chrFilter, const int16_t **chrUSrc,
482                                  const int16_t **chrVSrc,
483                                  int chrFilterSize, const int16_t **alpSrc,
484                                  uint8_t *dest, int dstW, int dstY)
485 {
486     x86_reg dummy=0;
487     x86_reg dstW_reg = dstW;
488     x86_reg uv_off = c->uv_offx2;
489
490     YSCALEYUV2PACKEDX
491     YSCALEYUV2RGBX
492     "pxor %%mm7, %%mm7 \n\t"
493     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
494 #ifdef DITHER1XBPP
495     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
496     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
497     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
498 #endif
499     WRITERGB16(%4, %5, %%REGa)
500     YSCALEYUV2PACKEDX_END
501 }
502
503 #define REAL_WRITERGB15(dst, dstw, index) \
504     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
505     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
506     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
507     "psrlq           $3, %%mm2  \n\t"\
508     "psrlq           $1, %%mm5  \n\t"\
509 \
510     "movq         %%mm2, %%mm1  \n\t"\
511     "movq         %%mm4, %%mm3  \n\t"\
512 \
513     "punpcklbw    %%mm7, %%mm3  \n\t"\
514     "punpcklbw    %%mm5, %%mm2  \n\t"\
515     "punpckhbw    %%mm7, %%mm4  \n\t"\
516     "punpckhbw    %%mm5, %%mm1  \n\t"\
517 \
518     "psllq           $2, %%mm3  \n\t"\
519     "psllq           $2, %%mm4  \n\t"\
520 \
521     "por          %%mm3, %%mm2  \n\t"\
522     "por          %%mm4, %%mm1  \n\t"\
523 \
524     MOVNTQ(%%mm2,  (dst, index, 2))\
525     MOVNTQ(%%mm1, 8(dst, index, 2))\
526 \
527     "add             $8, "#index"   \n\t"\
528     "cmp        "#dstw", "#index"   \n\t"\
529     " jb             1b             \n\t"
530 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
531
532 static void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,
533                                     const int16_t **lumSrc, int lumFilterSize,
534                                     const int16_t *chrFilter, const int16_t **chrUSrc,
535                                     const int16_t **chrVSrc,
536                                     int chrFilterSize, const int16_t **alpSrc,
537                                     uint8_t *dest, int dstW, int dstY)
538 {
539     x86_reg dummy=0;
540     x86_reg dstW_reg = dstW;
541     x86_reg uv_off = c->uv_offx2;
542
543     YSCALEYUV2PACKEDX_ACCURATE
544     YSCALEYUV2RGBX
545     "pxor %%mm7, %%mm7 \n\t"
546     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
547 #ifdef DITHER1XBPP
548     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
549     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
550     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
551 #endif
552     WRITERGB15(%4, %5, %%REGa)
553     YSCALEYUV2PACKEDX_END
554 }
555
556 static void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,
557                                  const int16_t **lumSrc, int lumFilterSize,
558                                  const int16_t *chrFilter, const int16_t **chrUSrc,
559                                  const int16_t **chrVSrc,
560                                  int chrFilterSize, const int16_t **alpSrc,
561                                  uint8_t *dest, int dstW, int dstY)
562 {
563     x86_reg dummy=0;
564     x86_reg dstW_reg = dstW;
565     x86_reg uv_off = c->uv_offx2;
566
567     YSCALEYUV2PACKEDX
568     YSCALEYUV2RGBX
569     "pxor %%mm7, %%mm7 \n\t"
570     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
571 #ifdef DITHER1XBPP
572     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
573     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
574     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
575 #endif
576     WRITERGB15(%4, %5, %%REGa)
577     YSCALEYUV2PACKEDX_END
578 }
579
580 #define WRITEBGR24MMX(dst, dstw, index) \
581     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
582     "movq      %%mm2, %%mm1     \n\t" /* B */\
583     "movq      %%mm5, %%mm6     \n\t" /* R */\
584     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
585     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
586     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
587     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
588     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
589     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
590     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
591     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
592     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
593     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
594 \
595     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
596     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
597     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
598     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
599 \
600     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
601     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
602     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
603     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
604 \
605     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
606     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
607     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
608     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
609 \
610     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
611     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
612     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
613     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
614     MOVNTQ(%%mm0, (dst))\
615 \
616     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
617     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
618     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
619     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
620     MOVNTQ(%%mm6, 8(dst))\
621 \
622     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
623     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
624     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
625     MOVNTQ(%%mm5, 16(dst))\
626 \
627     "add         $24, "#dst"    \n\t"\
628 \
629     "add          $8, "#index"  \n\t"\
630     "cmp     "#dstw", "#index"  \n\t"\
631     " jb          1b            \n\t"
632
633 #define WRITEBGR24MMX2(dst, dstw, index) \
634     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
635     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
636     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
637     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
638     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
639     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
640 \
641     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
642     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
643     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
644 \
645     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
646     "por    %%mm1, %%mm6        \n\t"\
647     "por    %%mm3, %%mm6        \n\t"\
648     MOVNTQ(%%mm6, (dst))\
649 \
650     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
651     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
652     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
653     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
654 \
655     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
656     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
657     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
658 \
659     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
660     "por    %%mm3, %%mm6        \n\t"\
661     MOVNTQ(%%mm6, 8(dst))\
662 \
663     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
664     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
665     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
666 \
667     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
668     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
669     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
670 \
671     "por    %%mm1, %%mm3        \n\t"\
672     "por    %%mm3, %%mm6        \n\t"\
673     MOVNTQ(%%mm6, 16(dst))\
674 \
675     "add      $24, "#dst"       \n\t"\
676 \
677     "add       $8, "#index"     \n\t"\
678     "cmp  "#dstw", "#index"     \n\t"\
679     " jb       1b               \n\t"
680
681 #if COMPILE_TEMPLATE_MMX2
682 #undef WRITEBGR24
683 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
684 #else
685 #undef WRITEBGR24
686 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
687 #endif
688
689 static void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,
690                                    const int16_t **lumSrc, int lumFilterSize,
691                                    const int16_t *chrFilter, const int16_t **chrUSrc,
692                                    const int16_t **chrVSrc,
693                                    int chrFilterSize, const int16_t **alpSrc,
694                                    uint8_t *dest, int dstW, int dstY)
695 {
696     x86_reg dummy=0;
697     x86_reg dstW_reg = dstW;
698     x86_reg uv_off = c->uv_offx2;
699
700     YSCALEYUV2PACKEDX_ACCURATE
701     YSCALEYUV2RGBX
702     "pxor %%mm7, %%mm7 \n\t"
703     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
704     "add %4, %%"REG_c"                        \n\t"
705     WRITEBGR24(%%REGc, %5, %%REGa)
706     :: "r" (&c->redDither),
707        "m" (dummy), "m" (dummy), "m" (dummy),
708        "r" (dest), "m" (dstW_reg), "m"(uv_off)
709     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
710     );
711 }
712
713 static void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,
714                                 const int16_t **lumSrc, int lumFilterSize,
715                                 const int16_t *chrFilter, const int16_t **chrUSrc,
716                                 const int16_t **chrVSrc,
717                                 int chrFilterSize, const int16_t **alpSrc,
718                                 uint8_t *dest, int dstW, int dstY)
719 {
720     x86_reg dummy=0;
721     x86_reg dstW_reg = dstW;
722     x86_reg uv_off = c->uv_offx2;
723
724     YSCALEYUV2PACKEDX
725     YSCALEYUV2RGBX
726     "pxor                    %%mm7, %%mm7       \n\t"
727     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
728     "add                        %4, %%"REG_c"   \n\t"
729     WRITEBGR24(%%REGc, %5, %%REGa)
730     :: "r" (&c->redDither),
731        "m" (dummy), "m" (dummy), "m" (dummy),
732        "r" (dest),  "m" (dstW_reg), "m"(uv_off)
733     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
734     );
735 }
736
737 #define REAL_WRITEYUY2(dst, dstw, index) \
738     "packuswb  %%mm3, %%mm3     \n\t"\
739     "packuswb  %%mm4, %%mm4     \n\t"\
740     "packuswb  %%mm7, %%mm1     \n\t"\
741     "punpcklbw %%mm4, %%mm3     \n\t"\
742     "movq      %%mm1, %%mm7     \n\t"\
743     "punpcklbw %%mm3, %%mm1     \n\t"\
744     "punpckhbw %%mm3, %%mm7     \n\t"\
745 \
746     MOVNTQ(%%mm1, (dst, index, 2))\
747     MOVNTQ(%%mm7, 8(dst, index, 2))\
748 \
749     "add          $8, "#index"  \n\t"\
750     "cmp     "#dstw", "#index"  \n\t"\
751     " jb          1b            \n\t"
752 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
753
754 static void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,
755                                      const int16_t **lumSrc, int lumFilterSize,
756                                      const int16_t *chrFilter, const int16_t **chrUSrc,
757                                      const int16_t **chrVSrc,
758                                      int chrFilterSize, const int16_t **alpSrc,
759                                      uint8_t *dest, int dstW, int dstY)
760 {
761     x86_reg dummy=0;
762     x86_reg dstW_reg = dstW;
763     x86_reg uv_off = c->uv_offx2;
764
765     YSCALEYUV2PACKEDX_ACCURATE
766     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
767     "psraw $3, %%mm3    \n\t"
768     "psraw $3, %%mm4    \n\t"
769     "psraw $3, %%mm1    \n\t"
770     "psraw $3, %%mm7    \n\t"
771     WRITEYUY2(%4, %5, %%REGa)
772     YSCALEYUV2PACKEDX_END
773 }
774
775 static void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,
776                                   const int16_t **lumSrc, int lumFilterSize,
777                                   const int16_t *chrFilter, const int16_t **chrUSrc,
778                                   const int16_t **chrVSrc,
779                                   int chrFilterSize, const int16_t **alpSrc,
780                                   uint8_t *dest, int dstW, int dstY)
781 {
782     x86_reg dummy=0;
783     x86_reg dstW_reg = dstW;
784     x86_reg uv_off = c->uv_offx2;
785
786     YSCALEYUV2PACKEDX
787     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
788     "psraw $3, %%mm3    \n\t"
789     "psraw $3, %%mm4    \n\t"
790     "psraw $3, %%mm1    \n\t"
791     "psraw $3, %%mm7    \n\t"
792     WRITEYUY2(%4, %5, %%REGa)
793     YSCALEYUV2PACKEDX_END
794 }
795
796 #define REAL_YSCALEYUV2RGB_UV(index, c) \
797     "xor            "#index", "#index"  \n\t"\
798     ".p2align              4            \n\t"\
799     "1:                                 \n\t"\
800     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
801     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
802     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
803     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
804     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
805     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
806     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
807     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
808     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
809     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
810     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
811     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
812     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
813     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
814     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
815     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
816     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
817     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
818     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
819     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
820     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
821     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
822
823 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
824     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
825     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
826     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
827     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
828     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
829     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
830     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
831     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
832     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
833     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
834     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
835     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
836
837 #define REAL_YSCALEYUV2RGB_COEFF(c) \
838     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
839     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
840     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
841     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
842     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
843     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
844     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
845     "paddw             %%mm3, %%mm4     \n\t"\
846     "movq              %%mm2, %%mm0     \n\t"\
847     "movq              %%mm5, %%mm6     \n\t"\
848     "movq              %%mm4, %%mm3     \n\t"\
849     "punpcklwd         %%mm2, %%mm2     \n\t"\
850     "punpcklwd         %%mm5, %%mm5     \n\t"\
851     "punpcklwd         %%mm4, %%mm4     \n\t"\
852     "paddw             %%mm1, %%mm2     \n\t"\
853     "paddw             %%mm1, %%mm5     \n\t"\
854     "paddw             %%mm1, %%mm4     \n\t"\
855     "punpckhwd         %%mm0, %%mm0     \n\t"\
856     "punpckhwd         %%mm6, %%mm6     \n\t"\
857     "punpckhwd         %%mm3, %%mm3     \n\t"\
858     "paddw             %%mm7, %%mm0     \n\t"\
859     "paddw             %%mm7, %%mm6     \n\t"\
860     "paddw             %%mm7, %%mm3     \n\t"\
861     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
862     "packuswb          %%mm0, %%mm2     \n\t"\
863     "packuswb          %%mm6, %%mm5     \n\t"\
864     "packuswb          %%mm3, %%mm4     \n\t"\
865
866 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
867
868 #define YSCALEYUV2RGB(index, c) \
869     REAL_YSCALEYUV2RGB_UV(index, c) \
870     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
871     REAL_YSCALEYUV2RGB_COEFF(c)
872
873 /**
874  * vertical bilinear scale YV12 to RGB
875  */
876 static void RENAME(yuv2rgb32_2)(SwsContext *c, const int16_t *buf[2],
877                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
878                                 const int16_t *abuf[2], uint8_t *dest,
879                                 int dstW, int yalpha, int uvalpha, int y)
880 {
881     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
882                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
883
884     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
885         const int16_t *abuf0 = abuf[0], *abuf1 = abuf[1];
886 #if ARCH_X86_64
887         __asm__ volatile(
888             YSCALEYUV2RGB(%%r8, %5)
889             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
890             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
891             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
892             "packuswb            %%mm7, %%mm1       \n\t"
893             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
894             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "r" (dest),
895                "a" (&c->redDither),
896                "r" (abuf0), "r" (abuf1)
897             : "%r8"
898         );
899 #else
900         c->u_temp=(intptr_t)abuf0;
901         c->v_temp=(intptr_t)abuf1;
902         __asm__ volatile(
903             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
904             "mov        %4, %%"REG_b"               \n\t"
905             "push %%"REG_BP"                        \n\t"
906             YSCALEYUV2RGB(%%REGBP, %5)
907             "push                   %0              \n\t"
908             "push                   %1              \n\t"
909             "mov          "U_TEMP"(%5), %0          \n\t"
910             "mov          "V_TEMP"(%5), %1          \n\t"
911             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
912             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
913             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
914             "packuswb            %%mm7, %%mm1       \n\t"
915             "pop                    %1              \n\t"
916             "pop                    %0              \n\t"
917             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
918             "pop %%"REG_BP"                         \n\t"
919             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
920             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
921                "a" (&c->redDither)
922         );
923 #endif
924     } else {
925         __asm__ volatile(
926             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
927             "mov        %4, %%"REG_b"               \n\t"
928             "push %%"REG_BP"                        \n\t"
929             YSCALEYUV2RGB(%%REGBP, %5)
930             "pcmpeqd %%mm7, %%mm7                   \n\t"
931             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
932             "pop %%"REG_BP"                         \n\t"
933             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
934             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
935                "a" (&c->redDither)
936         );
937     }
938 }
939
940 static void RENAME(yuv2bgr24_2)(SwsContext *c, const int16_t *buf[2],
941                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
942                                 const int16_t *abuf[2], uint8_t *dest,
943                                 int dstW, int yalpha, int uvalpha, int y)
944 {
945     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
946                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
947
948     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
949     __asm__ volatile(
950         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
951         "mov        %4, %%"REG_b"               \n\t"
952         "push %%"REG_BP"                        \n\t"
953         YSCALEYUV2RGB(%%REGBP, %5)
954         "pxor    %%mm7, %%mm7                   \n\t"
955         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
956         "pop %%"REG_BP"                         \n\t"
957         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
958         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
959            "a" (&c->redDither)
960     );
961 }
962
963 static void RENAME(yuv2rgb555_2)(SwsContext *c, const int16_t *buf[2],
964                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
965                                  const int16_t *abuf[2], uint8_t *dest,
966                                  int dstW, int yalpha, int uvalpha, int y)
967 {
968     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
969                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
970
971     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
972     __asm__ volatile(
973         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
974         "mov        %4, %%"REG_b"               \n\t"
975         "push %%"REG_BP"                        \n\t"
976         YSCALEYUV2RGB(%%REGBP, %5)
977         "pxor    %%mm7, %%mm7                   \n\t"
978         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
979 #ifdef DITHER1XBPP
980         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
981         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
982         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
983 #endif
984         WRITERGB15(%%REGb, 8280(%5), %%REGBP)
985         "pop %%"REG_BP"                         \n\t"
986         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
987         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
988            "a" (&c->redDither)
989     );
990 }
991
992 static void RENAME(yuv2rgb565_2)(SwsContext *c, const int16_t *buf[2],
993                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
994                                  const int16_t *abuf[2], uint8_t *dest,
995                                  int dstW, int yalpha, int uvalpha, int y)
996 {
997     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
998                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
999
1000     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1001     __asm__ volatile(
1002         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1003         "mov        %4, %%"REG_b"               \n\t"
1004         "push %%"REG_BP"                        \n\t"
1005         YSCALEYUV2RGB(%%REGBP, %5)
1006         "pxor    %%mm7, %%mm7                   \n\t"
1007         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1008 #ifdef DITHER1XBPP
1009         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1010         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1011         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1012 #endif
1013         WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1014         "pop %%"REG_BP"                         \n\t"
1015         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1016         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1017            "a" (&c->redDither)
1018     );
1019 }
1020
1021 #define REAL_YSCALEYUV2PACKED(index, c) \
1022     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
1023     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
1024     "psraw                $3, %%mm0                           \n\t"\
1025     "psraw                $3, %%mm1                           \n\t"\
1026     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1027     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
1028     "xor            "#index", "#index"                        \n\t"\
1029     ".p2align              4            \n\t"\
1030     "1:                                 \n\t"\
1031     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1032     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1033     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1034     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1035     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1036     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1037     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
1038     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
1039     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
1040     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
1041     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
1042     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1043     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1044     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
1045     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
1046     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
1047     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
1048     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
1049     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
1050     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
1051     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
1052     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1053     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
1054     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1055     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1056     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1057     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
1058
1059 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
1060
1061 static void RENAME(yuv2yuyv422_2)(SwsContext *c, const int16_t *buf[2],
1062                                   const int16_t *ubuf[2], const int16_t *vbuf[2],
1063                                   const int16_t *abuf[2], uint8_t *dest,
1064                                   int dstW, int yalpha, int uvalpha, int y)
1065 {
1066     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
1067                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1068
1069     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1070     __asm__ volatile(
1071         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1072         "mov %4, %%"REG_b"                        \n\t"
1073         "push %%"REG_BP"                        \n\t"
1074         YSCALEYUV2PACKED(%%REGBP, %5)
1075         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1076         "pop %%"REG_BP"                         \n\t"
1077         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1078         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1079            "a" (&c->redDither)
1080     );
1081 }
1082
1083 #define REAL_YSCALEYUV2RGB1(index, c) \
1084     "xor            "#index", "#index"  \n\t"\
1085     ".p2align              4            \n\t"\
1086     "1:                                 \n\t"\
1087     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1088     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1089     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1090     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1091     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
1092     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
1093     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1094     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1095     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1096     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1097     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1098     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1099     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1100     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1101     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1102     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1103     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1104     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1105     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1106     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1107     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1108     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1109     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1110     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1111     "paddw             %%mm3, %%mm4     \n\t"\
1112     "movq              %%mm2, %%mm0     \n\t"\
1113     "movq              %%mm5, %%mm6     \n\t"\
1114     "movq              %%mm4, %%mm3     \n\t"\
1115     "punpcklwd         %%mm2, %%mm2     \n\t"\
1116     "punpcklwd         %%mm5, %%mm5     \n\t"\
1117     "punpcklwd         %%mm4, %%mm4     \n\t"\
1118     "paddw             %%mm1, %%mm2     \n\t"\
1119     "paddw             %%mm1, %%mm5     \n\t"\
1120     "paddw             %%mm1, %%mm4     \n\t"\
1121     "punpckhwd         %%mm0, %%mm0     \n\t"\
1122     "punpckhwd         %%mm6, %%mm6     \n\t"\
1123     "punpckhwd         %%mm3, %%mm3     \n\t"\
1124     "paddw             %%mm7, %%mm0     \n\t"\
1125     "paddw             %%mm7, %%mm6     \n\t"\
1126     "paddw             %%mm7, %%mm3     \n\t"\
1127     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1128     "packuswb          %%mm0, %%mm2     \n\t"\
1129     "packuswb          %%mm6, %%mm5     \n\t"\
1130     "packuswb          %%mm3, %%mm4     \n\t"\
1131
1132 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
1133
1134 // do vertical chrominance interpolation
1135 #define REAL_YSCALEYUV2RGB1b(index, c) \
1136     "xor            "#index", "#index"  \n\t"\
1137     ".p2align              4            \n\t"\
1138     "1:                                 \n\t"\
1139     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1140     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1141     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1142     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1143     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1144     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1145     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1146     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1147     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
1148     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
1149     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1150     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1151     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1152     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1153     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1154     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1155     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1156     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1157     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1158     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1159     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1160     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1161     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1162     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1163     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1164     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1165     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1166     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1167     "paddw             %%mm3, %%mm4     \n\t"\
1168     "movq              %%mm2, %%mm0     \n\t"\
1169     "movq              %%mm5, %%mm6     \n\t"\
1170     "movq              %%mm4, %%mm3     \n\t"\
1171     "punpcklwd         %%mm2, %%mm2     \n\t"\
1172     "punpcklwd         %%mm5, %%mm5     \n\t"\
1173     "punpcklwd         %%mm4, %%mm4     \n\t"\
1174     "paddw             %%mm1, %%mm2     \n\t"\
1175     "paddw             %%mm1, %%mm5     \n\t"\
1176     "paddw             %%mm1, %%mm4     \n\t"\
1177     "punpckhwd         %%mm0, %%mm0     \n\t"\
1178     "punpckhwd         %%mm6, %%mm6     \n\t"\
1179     "punpckhwd         %%mm3, %%mm3     \n\t"\
1180     "paddw             %%mm7, %%mm0     \n\t"\
1181     "paddw             %%mm7, %%mm6     \n\t"\
1182     "paddw             %%mm7, %%mm3     \n\t"\
1183     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1184     "packuswb          %%mm0, %%mm2     \n\t"\
1185     "packuswb          %%mm6, %%mm5     \n\t"\
1186     "packuswb          %%mm3, %%mm4     \n\t"\
1187
1188 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
1189
1190 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
1191     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
1192     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
1193     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
1194     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
1195     "packuswb          %%mm1, %%mm7     \n\t"
1196 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
1197
1198 /**
1199  * YV12 to RGB without scaling or interpolating
1200  */
1201 static void RENAME(yuv2rgb32_1)(SwsContext *c, const int16_t *buf0,
1202                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1203                                 const int16_t *abuf0, uint8_t *dest,
1204                                 int dstW, int uvalpha, int y)
1205 {
1206     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1207     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1208
1209     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1210         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1211             __asm__ volatile(
1212                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1213                 "mov        %4, %%"REG_b"               \n\t"
1214                 "push %%"REG_BP"                        \n\t"
1215                 YSCALEYUV2RGB1(%%REGBP, %5)
1216                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1217                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1218                 "pop %%"REG_BP"                         \n\t"
1219                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1220                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1221                    "a" (&c->redDither)
1222             );
1223         } else {
1224             __asm__ volatile(
1225                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1226                 "mov        %4, %%"REG_b"               \n\t"
1227                 "push %%"REG_BP"                        \n\t"
1228                 YSCALEYUV2RGB1(%%REGBP, %5)
1229                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1230                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1231                 "pop %%"REG_BP"                         \n\t"
1232                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1233                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1234                    "a" (&c->redDither)
1235             );
1236         }
1237     } else {
1238         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1239             __asm__ volatile(
1240                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1241                 "mov        %4, %%"REG_b"               \n\t"
1242                 "push %%"REG_BP"                        \n\t"
1243                 YSCALEYUV2RGB1b(%%REGBP, %5)
1244                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1245                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1246                 "pop %%"REG_BP"                         \n\t"
1247                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1248                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1249                    "a" (&c->redDither)
1250             );
1251         } else {
1252             __asm__ volatile(
1253                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1254                 "mov        %4, %%"REG_b"               \n\t"
1255                 "push %%"REG_BP"                        \n\t"
1256                 YSCALEYUV2RGB1b(%%REGBP, %5)
1257                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1258                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1259                 "pop %%"REG_BP"                         \n\t"
1260                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1261                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1262                    "a" (&c->redDither)
1263             );
1264         }
1265     }
1266 }
1267
1268 static void RENAME(yuv2bgr24_1)(SwsContext *c, const int16_t *buf0,
1269                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1270                                 const int16_t *abuf0, uint8_t *dest,
1271                                 int dstW, int uvalpha, int y)
1272 {
1273     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1274     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1275
1276     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1277         __asm__ volatile(
1278             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1279             "mov        %4, %%"REG_b"               \n\t"
1280             "push %%"REG_BP"                        \n\t"
1281             YSCALEYUV2RGB1(%%REGBP, %5)
1282             "pxor    %%mm7, %%mm7                   \n\t"
1283             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1284             "pop %%"REG_BP"                         \n\t"
1285             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1286             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1287                "a" (&c->redDither)
1288         );
1289     } else {
1290         __asm__ volatile(
1291             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1292             "mov        %4, %%"REG_b"               \n\t"
1293             "push %%"REG_BP"                        \n\t"
1294             YSCALEYUV2RGB1b(%%REGBP, %5)
1295             "pxor    %%mm7, %%mm7                   \n\t"
1296             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1297             "pop %%"REG_BP"                         \n\t"
1298             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1299             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1300                "a" (&c->redDither)
1301         );
1302     }
1303 }
1304
1305 static void RENAME(yuv2rgb555_1)(SwsContext *c, const int16_t *buf0,
1306                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1307                                  const int16_t *abuf0, uint8_t *dest,
1308                                  int dstW, int uvalpha, int y)
1309 {
1310     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1311     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1312
1313     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1314         __asm__ volatile(
1315             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1316             "mov        %4, %%"REG_b"               \n\t"
1317             "push %%"REG_BP"                        \n\t"
1318             YSCALEYUV2RGB1(%%REGBP, %5)
1319             "pxor    %%mm7, %%mm7                   \n\t"
1320             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1321 #ifdef DITHER1XBPP
1322             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1323             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1324             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1325 #endif
1326             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1327             "pop %%"REG_BP"                         \n\t"
1328             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1329             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1330                "a" (&c->redDither)
1331         );
1332     } else {
1333         __asm__ volatile(
1334             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1335             "mov        %4, %%"REG_b"               \n\t"
1336             "push %%"REG_BP"                        \n\t"
1337             YSCALEYUV2RGB1b(%%REGBP, %5)
1338             "pxor    %%mm7, %%mm7                   \n\t"
1339             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1340 #ifdef DITHER1XBPP
1341             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1342             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1343             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1344 #endif
1345             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1346             "pop %%"REG_BP"                         \n\t"
1347             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1348             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1349                "a" (&c->redDither)
1350         );
1351     }
1352 }
1353
1354 static void RENAME(yuv2rgb565_1)(SwsContext *c, const int16_t *buf0,
1355                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1356                                  const int16_t *abuf0, uint8_t *dest,
1357                                  int dstW, int uvalpha, int y)
1358 {
1359     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1360     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1361
1362     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1363         __asm__ volatile(
1364             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1365             "mov        %4, %%"REG_b"               \n\t"
1366             "push %%"REG_BP"                        \n\t"
1367             YSCALEYUV2RGB1(%%REGBP, %5)
1368             "pxor    %%mm7, %%mm7                   \n\t"
1369             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1370 #ifdef DITHER1XBPP
1371             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1372             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1373             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1374 #endif
1375             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1376             "pop %%"REG_BP"                         \n\t"
1377             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1378             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1379                "a" (&c->redDither)
1380         );
1381     } else {
1382         __asm__ volatile(
1383             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1384             "mov        %4, %%"REG_b"               \n\t"
1385             "push %%"REG_BP"                        \n\t"
1386             YSCALEYUV2RGB1b(%%REGBP, %5)
1387             "pxor    %%mm7, %%mm7                   \n\t"
1388             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1389 #ifdef DITHER1XBPP
1390             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1391             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1392             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1393 #endif
1394             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1395             "pop %%"REG_BP"                         \n\t"
1396             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1397             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1398                "a" (&c->redDither)
1399         );
1400     }
1401 }
1402
1403 #define REAL_YSCALEYUV2PACKED1(index, c) \
1404     "xor            "#index", "#index"  \n\t"\
1405     ".p2align              4            \n\t"\
1406     "1:                                 \n\t"\
1407     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1408     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1409     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1410     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1411     "psraw                $7, %%mm3     \n\t" \
1412     "psraw                $7, %%mm4     \n\t" \
1413     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1414     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1415     "psraw                $7, %%mm1     \n\t" \
1416     "psraw                $7, %%mm7     \n\t" \
1417
1418 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
1419
1420 #define REAL_YSCALEYUV2PACKED1b(index, c) \
1421     "xor "#index", "#index"             \n\t"\
1422     ".p2align              4            \n\t"\
1423     "1:                                 \n\t"\
1424     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1425     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1426     "add           "UV_OFFx2"("#c"), "#index"  \n\t" \
1427     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1428     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1429     "sub           "UV_OFFx2"("#c"), "#index"  \n\t" \
1430     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1431     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1432     "psrlw                $8, %%mm3     \n\t" \
1433     "psrlw                $8, %%mm4     \n\t" \
1434     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1435     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1436     "psraw                $7, %%mm1     \n\t" \
1437     "psraw                $7, %%mm7     \n\t"
1438 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
1439
1440 static void RENAME(yuv2yuyv422_1)(SwsContext *c, const int16_t *buf0,
1441                                   const int16_t *ubuf[2], const int16_t *bguf[2],
1442                                   const int16_t *abuf0, uint8_t *dest,
1443                                   int dstW, int uvalpha, int y)
1444 {
1445     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1446     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1447
1448     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1449         __asm__ volatile(
1450             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1451             "mov        %4, %%"REG_b"               \n\t"
1452             "push %%"REG_BP"                        \n\t"
1453             YSCALEYUV2PACKED1(%%REGBP, %5)
1454             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1455             "pop %%"REG_BP"                         \n\t"
1456             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1457             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1458                "a" (&c->redDither)
1459         );
1460     } else {
1461         __asm__ volatile(
1462             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1463             "mov        %4, %%"REG_b"               \n\t"
1464             "push %%"REG_BP"                        \n\t"
1465             YSCALEYUV2PACKED1b(%%REGBP, %5)
1466             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1467             "pop %%"REG_BP"                         \n\t"
1468             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1469             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1470                "a" (&c->redDither)
1471         );
1472     }
1473 }
1474
1475 #if !COMPILE_TEMPLATE_MMX2
1476 //FIXME yuy2* can read up to 7 samples too much
1477
1478 static void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src, const uint8_t *unused1, const uint8_t *unused2,
1479                             int width, uint32_t *unused)
1480 {
1481     __asm__ volatile(
1482         "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1483         "mov                    %0, %%"REG_a"       \n\t"
1484         "1:                                         \n\t"
1485         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1486         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1487         "pand                %%mm2, %%mm0           \n\t"
1488         "pand                %%mm2, %%mm1           \n\t"
1489         "packuswb            %%mm1, %%mm0           \n\t"
1490         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1491         "add                    $8, %%"REG_a"       \n\t"
1492         " js                    1b                  \n\t"
1493         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1494         : "%"REG_a
1495     );
1496 }
1497
1498 static void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV,
1499                              const uint8_t *unused1, const uint8_t *src1, const uint8_t *src2,
1500                              int width, uint32_t *unused)
1501 {
1502     __asm__ volatile(
1503         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1504         "mov                    %0, %%"REG_a"       \n\t"
1505         "1:                                         \n\t"
1506         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1507         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1508         "psrlw                  $8, %%mm0           \n\t"
1509         "psrlw                  $8, %%mm1           \n\t"
1510         "packuswb            %%mm1, %%mm0           \n\t"
1511         "movq                %%mm0, %%mm1           \n\t"
1512         "psrlw                  $8, %%mm0           \n\t"
1513         "pand                %%mm4, %%mm1           \n\t"
1514         "packuswb            %%mm0, %%mm0           \n\t"
1515         "packuswb            %%mm1, %%mm1           \n\t"
1516         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1517         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1518         "add                    $4, %%"REG_a"       \n\t"
1519         " js                    1b                  \n\t"
1520         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1521         : "%"REG_a
1522     );
1523     assert(src1 == src2);
1524 }
1525
1526 /* This is almost identical to the previous, end exists only because
1527  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1528 static void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src, const uint8_t *unused1, const uint8_t *unused2,
1529                             int width, uint32_t *unused)
1530 {
1531     __asm__ volatile(
1532         "mov                  %0, %%"REG_a"         \n\t"
1533         "1:                                         \n\t"
1534         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1535         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1536         "psrlw                $8, %%mm0             \n\t"
1537         "psrlw                $8, %%mm1             \n\t"
1538         "packuswb          %%mm1, %%mm0             \n\t"
1539         "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1540         "add                  $8, %%"REG_a"         \n\t"
1541         " js                  1b                    \n\t"
1542         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1543         : "%"REG_a
1544     );
1545 }
1546
1547 static void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV,
1548                              const uint8_t *unused1, const uint8_t *src1, const uint8_t *src2,
1549                              int width, uint32_t *unused)
1550 {
1551     __asm__ volatile(
1552         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1553         "mov                    %0, %%"REG_a"       \n\t"
1554         "1:                                         \n\t"
1555         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1556         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1557         "pand                %%mm4, %%mm0           \n\t"
1558         "pand                %%mm4, %%mm1           \n\t"
1559         "packuswb            %%mm1, %%mm0           \n\t"
1560         "movq                %%mm0, %%mm1           \n\t"
1561         "psrlw                  $8, %%mm0           \n\t"
1562         "pand                %%mm4, %%mm1           \n\t"
1563         "packuswb            %%mm0, %%mm0           \n\t"
1564         "packuswb            %%mm1, %%mm1           \n\t"
1565         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1566         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1567         "add                    $4, %%"REG_a"       \n\t"
1568         " js                    1b                  \n\t"
1569         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1570         : "%"REG_a
1571     );
1572     assert(src1 == src2);
1573 }
1574
1575 static av_always_inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1576                                               const uint8_t *src, int width)
1577 {
1578     __asm__ volatile(
1579         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1580         "mov                    %0, %%"REG_a"       \n\t"
1581         "1:                                         \n\t"
1582         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1583         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1584         "movq                %%mm0, %%mm2           \n\t"
1585         "movq                %%mm1, %%mm3           \n\t"
1586         "pand                %%mm4, %%mm0           \n\t"
1587         "pand                %%mm4, %%mm1           \n\t"
1588         "psrlw                  $8, %%mm2           \n\t"
1589         "psrlw                  $8, %%mm3           \n\t"
1590         "packuswb            %%mm1, %%mm0           \n\t"
1591         "packuswb            %%mm3, %%mm2           \n\t"
1592         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1593         "movq                %%mm2, (%3, %%"REG_a") \n\t"
1594         "add                    $8, %%"REG_a"       \n\t"
1595         " js                    1b                  \n\t"
1596         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1597         : "%"REG_a
1598     );
1599 }
1600
1601 static void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1602                              const uint8_t *unused1, const uint8_t *src1, const uint8_t *src2,
1603                              int width, uint32_t *unused)
1604 {
1605     RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1606 }
1607
1608 static void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1609                              const uint8_t *unused1, const uint8_t *src1, const uint8_t *src2,
1610                              int width, uint32_t *unused)
1611 {
1612     RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1613 }
1614 #endif /* !COMPILE_TEMPLATE_MMX2 */
1615
1616 static av_always_inline void RENAME(bgr24ToY_mmx)(int16_t *dst, const uint8_t *src,
1617                                                   int width, enum PixelFormat srcFormat)
1618 {
1619
1620     if(srcFormat == PIX_FMT_BGR24) {
1621         __asm__ volatile(
1622             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1623             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1624             :
1625         );
1626     } else {
1627         __asm__ volatile(
1628             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1629             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1630             :
1631         );
1632     }
1633
1634     __asm__ volatile(
1635         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1636         "mov                        %2, %%"REG_a"   \n\t"
1637         "pxor                    %%mm7, %%mm7       \n\t"
1638         "1:                                         \n\t"
1639         PREFETCH"               64(%0)              \n\t"
1640         "movd                     (%0), %%mm0       \n\t"
1641         "movd                    2(%0), %%mm1       \n\t"
1642         "movd                    6(%0), %%mm2       \n\t"
1643         "movd                    8(%0), %%mm3       \n\t"
1644         "add                       $12, %0          \n\t"
1645         "punpcklbw               %%mm7, %%mm0       \n\t"
1646         "punpcklbw               %%mm7, %%mm1       \n\t"
1647         "punpcklbw               %%mm7, %%mm2       \n\t"
1648         "punpcklbw               %%mm7, %%mm3       \n\t"
1649         "pmaddwd                 %%mm5, %%mm0       \n\t"
1650         "pmaddwd                 %%mm6, %%mm1       \n\t"
1651         "pmaddwd                 %%mm5, %%mm2       \n\t"
1652         "pmaddwd                 %%mm6, %%mm3       \n\t"
1653         "paddd                   %%mm1, %%mm0       \n\t"
1654         "paddd                   %%mm3, %%mm2       \n\t"
1655         "paddd                   %%mm4, %%mm0       \n\t"
1656         "paddd                   %%mm4, %%mm2       \n\t"
1657         "psrad                     $9, %%mm0       \n\t"
1658         "psrad                     $9, %%mm2       \n\t"
1659         "packssdw                %%mm2, %%mm0       \n\t"
1660         "movq                %%mm0, (%1, %%"REG_a") \n\t"
1661         "add                        $8, %%"REG_a"   \n\t"
1662         " js                        1b              \n\t"
1663     : "+r" (src)
1664     : "r" (dst+width), "g" ((x86_reg)-2*width)
1665     : "%"REG_a
1666     );
1667 }
1668
1669 static void RENAME(bgr24ToY)(int16_t *dst, const uint8_t *src, const uint8_t *unused1, const uint8_t *unused2,
1670                              int width, uint32_t *unused)
1671 {
1672     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1673 }
1674
1675 static void RENAME(rgb24ToY)(int16_t *dst, const uint8_t *src, const uint8_t *unused1, const uint8_t *unused2,
1676                              int width, uint32_t *unused)
1677 {
1678     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1679 }
1680
1681 static av_always_inline void RENAME(bgr24ToUV_mmx)(int16_t *dstU, int16_t *dstV,
1682                                                    const uint8_t *src, int width,
1683                                                    enum PixelFormat srcFormat)
1684 {
1685     __asm__ volatile(
1686         "movq                    24(%4), %%mm6       \n\t"
1687         "mov                        %3, %%"REG_a"   \n\t"
1688         "pxor                    %%mm7, %%mm7       \n\t"
1689         "1:                                         \n\t"
1690         PREFETCH"               64(%0)              \n\t"
1691         "movd                     (%0), %%mm0       \n\t"
1692         "movd                    2(%0), %%mm1       \n\t"
1693         "punpcklbw               %%mm7, %%mm0       \n\t"
1694         "punpcklbw               %%mm7, %%mm1       \n\t"
1695         "movq                    %%mm0, %%mm2       \n\t"
1696         "movq                    %%mm1, %%mm3       \n\t"
1697         "pmaddwd                  (%4), %%mm0       \n\t"
1698         "pmaddwd                 8(%4), %%mm1       \n\t"
1699         "pmaddwd                16(%4), %%mm2       \n\t"
1700         "pmaddwd                 %%mm6, %%mm3       \n\t"
1701         "paddd                   %%mm1, %%mm0       \n\t"
1702         "paddd                   %%mm3, %%mm2       \n\t"
1703
1704         "movd                    6(%0), %%mm1       \n\t"
1705         "movd                    8(%0), %%mm3       \n\t"
1706         "add                       $12, %0          \n\t"
1707         "punpcklbw               %%mm7, %%mm1       \n\t"
1708         "punpcklbw               %%mm7, %%mm3       \n\t"
1709         "movq                    %%mm1, %%mm4       \n\t"
1710         "movq                    %%mm3, %%mm5       \n\t"
1711         "pmaddwd                  (%4), %%mm1       \n\t"
1712         "pmaddwd                 8(%4), %%mm3       \n\t"
1713         "pmaddwd                16(%4), %%mm4       \n\t"
1714         "pmaddwd                 %%mm6, %%mm5       \n\t"
1715         "paddd                   %%mm3, %%mm1       \n\t"
1716         "paddd                   %%mm5, %%mm4       \n\t"
1717
1718         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1719         "paddd                   %%mm3, %%mm0       \n\t"
1720         "paddd                   %%mm3, %%mm2       \n\t"
1721         "paddd                   %%mm3, %%mm1       \n\t"
1722         "paddd                   %%mm3, %%mm4       \n\t"
1723         "psrad                     $9, %%mm0       \n\t"
1724         "psrad                     $9, %%mm2       \n\t"
1725         "psrad                     $9, %%mm1       \n\t"
1726         "psrad                     $9, %%mm4       \n\t"
1727         "packssdw                %%mm1, %%mm0       \n\t"
1728         "packssdw                %%mm4, %%mm2       \n\t"
1729         "movq                %%mm0, (%1, %%"REG_a") \n\t"
1730         "movq                %%mm2, (%2, %%"REG_a") \n\t"
1731         "add                        $8, %%"REG_a"   \n\t"
1732         " js                        1b              \n\t"
1733     : "+r" (src)
1734     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-2*width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1735     : "%"REG_a
1736     );
1737 }
1738
1739 static void RENAME(bgr24ToUV)(int16_t *dstU, int16_t *dstV,
1740                               const uint8_t *unused1, const uint8_t *src1, const uint8_t *src2,
1741                               int width, uint32_t *unused)
1742 {
1743     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1744     assert(src1 == src2);
1745 }
1746
1747 static void RENAME(rgb24ToUV)(int16_t *dstU, int16_t *dstV,
1748                               const uint8_t *unused1, const uint8_t *src1, const uint8_t *src2,
1749                               int width, uint32_t *unused)
1750 {
1751     assert(src1==src2);
1752     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1753 }
1754
1755 #if COMPILE_TEMPLATE_MMX2
1756 static void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
1757                                  int dstWidth, const uint8_t *src,
1758                                  int srcW, int xInc)
1759 {
1760     int16_t *filterPos = c->hLumFilterPos;
1761     int16_t *filter    = c->hLumFilter;
1762     void    *mmx2FilterCode= c->lumMmx2FilterCode;
1763     int i;
1764 #if defined(PIC)
1765     DECLARE_ALIGNED(8, uint64_t, ebxsave);
1766 #endif
1767
1768     __asm__ volatile(
1769 #if defined(PIC)
1770         "mov               %%"REG_b", %5        \n\t"
1771 #endif
1772         "pxor                  %%mm7, %%mm7     \n\t"
1773         "mov                      %0, %%"REG_c" \n\t"
1774         "mov                      %1, %%"REG_D" \n\t"
1775         "mov                      %2, %%"REG_d" \n\t"
1776         "mov                      %3, %%"REG_b" \n\t"
1777         "xor               %%"REG_a", %%"REG_a" \n\t" // i
1778         PREFETCH"        (%%"REG_c")            \n\t"
1779         PREFETCH"      32(%%"REG_c")            \n\t"
1780         PREFETCH"      64(%%"REG_c")            \n\t"
1781
1782 #if ARCH_X86_64
1783 #define CALL_MMX2_FILTER_CODE \
1784         "movl            (%%"REG_b"), %%esi     \n\t"\
1785         "call                    *%4            \n\t"\
1786         "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
1787         "add               %%"REG_S", %%"REG_c" \n\t"\
1788         "add               %%"REG_a", %%"REG_D" \n\t"\
1789         "xor               %%"REG_a", %%"REG_a" \n\t"\
1790
1791 #else
1792 #define CALL_MMX2_FILTER_CODE \
1793         "movl (%%"REG_b"), %%esi        \n\t"\
1794         "call         *%4                       \n\t"\
1795         "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
1796         "add               %%"REG_a", %%"REG_D" \n\t"\
1797         "xor               %%"REG_a", %%"REG_a" \n\t"\
1798
1799 #endif /* ARCH_X86_64 */
1800
1801         CALL_MMX2_FILTER_CODE
1802         CALL_MMX2_FILTER_CODE
1803         CALL_MMX2_FILTER_CODE
1804         CALL_MMX2_FILTER_CODE
1805         CALL_MMX2_FILTER_CODE
1806         CALL_MMX2_FILTER_CODE
1807         CALL_MMX2_FILTER_CODE
1808         CALL_MMX2_FILTER_CODE
1809
1810 #if defined(PIC)
1811         "mov                      %5, %%"REG_b" \n\t"
1812 #endif
1813         :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
1814            "m" (mmx2FilterCode)
1815 #if defined(PIC)
1816           ,"m" (ebxsave)
1817 #endif
1818         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
1819 #if !defined(PIC)
1820          ,"%"REG_b
1821 #endif
1822     );
1823
1824     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
1825         dst[i] = src[srcW-1]*128;
1826 }
1827
1828 static void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2,
1829                                  int dstWidth, const uint8_t *src1,
1830                                  const uint8_t *src2, int srcW, int xInc)
1831 {
1832     int16_t *filterPos = c->hChrFilterPos;
1833     int16_t *filter    = c->hChrFilter;
1834     void    *mmx2FilterCode= c->chrMmx2FilterCode;
1835     int i;
1836 #if defined(PIC)
1837     DECLARE_ALIGNED(8, uint64_t, ebxsave);
1838 #endif
1839
1840     __asm__ volatile(
1841 #if defined(PIC)
1842         "mov          %%"REG_b", %7         \n\t"
1843 #endif
1844         "pxor             %%mm7, %%mm7      \n\t"
1845         "mov                 %0, %%"REG_c"  \n\t"
1846         "mov                 %1, %%"REG_D"  \n\t"
1847         "mov                 %2, %%"REG_d"  \n\t"
1848         "mov                 %3, %%"REG_b"  \n\t"
1849         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
1850         PREFETCH"   (%%"REG_c")             \n\t"
1851         PREFETCH" 32(%%"REG_c")             \n\t"
1852         PREFETCH" 64(%%"REG_c")             \n\t"
1853
1854         CALL_MMX2_FILTER_CODE
1855         CALL_MMX2_FILTER_CODE
1856         CALL_MMX2_FILTER_CODE
1857         CALL_MMX2_FILTER_CODE
1858         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
1859         "mov                 %5, %%"REG_c"  \n\t" // src
1860         "mov                 %6, %%"REG_D"  \n\t" // buf2
1861         PREFETCH"   (%%"REG_c")             \n\t"
1862         PREFETCH" 32(%%"REG_c")             \n\t"
1863         PREFETCH" 64(%%"REG_c")             \n\t"
1864
1865         CALL_MMX2_FILTER_CODE
1866         CALL_MMX2_FILTER_CODE
1867         CALL_MMX2_FILTER_CODE
1868         CALL_MMX2_FILTER_CODE
1869
1870 #if defined(PIC)
1871         "mov %7, %%"REG_b"    \n\t"
1872 #endif
1873         :: "m" (src1), "m" (dst1), "m" (filter), "m" (filterPos),
1874            "m" (mmx2FilterCode), "m" (src2), "m"(dst2)
1875 #if defined(PIC)
1876           ,"m" (ebxsave)
1877 #endif
1878         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
1879 #if !defined(PIC)
1880          ,"%"REG_b
1881 #endif
1882     );
1883
1884     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
1885         dst1[i] = src1[srcW-1]*128;
1886         dst2[i] = src2[srcW-1]*128;
1887     }
1888 }
1889 #endif /* COMPILE_TEMPLATE_MMX2 */
1890
1891 static av_cold void RENAME(sws_init_swScale)(SwsContext *c)
1892 {
1893     enum PixelFormat srcFormat = c->srcFormat,
1894                      dstFormat = c->dstFormat;
1895
1896     if (!is16BPS(dstFormat) && !is9_OR_10BPS(dstFormat) && dstFormat != PIX_FMT_NV12
1897         && dstFormat != PIX_FMT_NV21 && !(c->flags & SWS_BITEXACT)) {
1898             if (c->flags & SWS_ACCURATE_RND) {
1899                 //c->yuv2yuv1 = RENAME(yuv2yuv1_ar    );
1900                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1901                     switch (c->dstFormat) {
1902                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;
1903                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;
1904                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;
1905                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;
1906                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;
1907                     default: break;
1908                     }
1909                 }
1910             } else {
1911                 //c->yuv2yuv1 = RENAME(yuv2yuv1    );
1912                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1913                     switch (c->dstFormat) {
1914                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;
1915                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;
1916                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;
1917                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;
1918                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;
1919                     default: break;
1920                     }
1921                 }
1922             }
1923         if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1924             switch (c->dstFormat) {
1925             case PIX_FMT_RGB32:
1926                 c->yuv2packed1 = RENAME(yuv2rgb32_1);
1927                 c->yuv2packed2 = RENAME(yuv2rgb32_2);
1928                 break;
1929             case PIX_FMT_BGR24:
1930                 c->yuv2packed1 = RENAME(yuv2bgr24_1);
1931                 c->yuv2packed2 = RENAME(yuv2bgr24_2);
1932                 break;
1933             case PIX_FMT_RGB555:
1934                 c->yuv2packed1 = RENAME(yuv2rgb555_1);
1935                 c->yuv2packed2 = RENAME(yuv2rgb555_2);
1936                 break;
1937             case PIX_FMT_RGB565:
1938                 c->yuv2packed1 = RENAME(yuv2rgb565_1);
1939                 c->yuv2packed2 = RENAME(yuv2rgb565_2);
1940                 break;
1941             case PIX_FMT_YUYV422:
1942                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);
1943                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);
1944                 break;
1945             default:
1946                 break;
1947             }
1948         }
1949     }
1950
1951     if (c->srcBpc == 8 && c->dstBpc <= 10) {
1952     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
1953 #if COMPILE_TEMPLATE_MMX2
1954     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
1955     {
1956         c->hyscale_fast = RENAME(hyscale_fast);
1957         c->hcscale_fast = RENAME(hcscale_fast);
1958     } else {
1959 #endif /* COMPILE_TEMPLATE_MMX2 */
1960         c->hyscale_fast = NULL;
1961         c->hcscale_fast = NULL;
1962 #if COMPILE_TEMPLATE_MMX2
1963     }
1964 #endif /* COMPILE_TEMPLATE_MMX2 */
1965     }
1966
1967 #if !COMPILE_TEMPLATE_MMX2
1968     switch(srcFormat) {
1969         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;
1970         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;
1971         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;
1972         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;
1973         default: break;
1974     }
1975 #endif /* !COMPILE_TEMPLATE_MMX2 */
1976     if (!c->chrSrcHSubSample) {
1977         switch(srcFormat) {
1978         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
1979         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
1980         default: break;
1981         }
1982     }
1983
1984     switch (srcFormat) {
1985 #if !COMPILE_TEMPLATE_MMX2
1986     case PIX_FMT_YUYV422  :
1987     case PIX_FMT_Y400A    : c->lumToYV12 = RENAME(yuy2ToY); break;
1988     case PIX_FMT_UYVY422  : c->lumToYV12 = RENAME(uyvyToY); break;
1989 #endif /* !COMPILE_TEMPLATE_MMX2 */
1990     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
1991     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
1992     default: break;
1993     }
1994 #if !COMPILE_TEMPLATE_MMX2
1995     if (c->alpPixBuf) {
1996         switch (srcFormat) {
1997         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;
1998         default: break;
1999         }
2000     }
2001 #endif /* !COMPILE_TEMPLATE_MMX2 */
2002 }