lavf doxy: add metadata docs to the main lavf group
[ffmpeg.git] / libswscale / x86 / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of Libav.
5  *
6  * Libav is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * Libav is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with Libav; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #undef REAL_MOVNTQ
22 #undef MOVNTQ
23 #undef PREFETCH
24
25 #if COMPILE_TEMPLATE_MMX2
26 #define PREFETCH "prefetchnta"
27 #else
28 #define PREFETCH  " # nop"
29 #endif
30
31 #if COMPILE_TEMPLATE_MMX2
32 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
33 #else
34 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
35 #endif
36 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
37
38 #define YSCALEYUV2PACKEDX_UV \
39     __asm__ volatile(\
40         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
41         ".p2align                      4                \n\t"\
42         "nop                                            \n\t"\
43         "1:                                             \n\t"\
44         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
45         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
46         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
47         "movq                      %%mm3, %%mm4         \n\t"\
48         ".p2align                      4                \n\t"\
49         "2:                                             \n\t"\
50         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
51         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
52         "add                          %6, %%"REG_S"     \n\t" \
53         "movq     (%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
54         "add                         $16, %%"REG_d"     \n\t"\
55         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
56         "pmulhw                    %%mm0, %%mm2         \n\t"\
57         "pmulhw                    %%mm0, %%mm5         \n\t"\
58         "paddw                     %%mm2, %%mm3         \n\t"\
59         "paddw                     %%mm5, %%mm4         \n\t"\
60         "test                  %%"REG_S", %%"REG_S"     \n\t"\
61         " jnz                         2b                \n\t"\
62
63 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
64     "lea                "offset"(%0), %%"REG_d"     \n\t"\
65     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
66     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
67     "movq                    "#dst1", "#dst2"       \n\t"\
68     ".p2align                      4                \n\t"\
69     "2:                                             \n\t"\
70     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
71     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
72     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
73     "add                         $16, %%"REG_d"            \n\t"\
74     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
75     "pmulhw                 "#coeff", "#src1"       \n\t"\
76     "pmulhw                 "#coeff", "#src2"       \n\t"\
77     "paddw                   "#src1", "#dst1"       \n\t"\
78     "paddw                   "#src2", "#dst2"       \n\t"\
79     "test                  %%"REG_S", %%"REG_S"     \n\t"\
80     " jnz                         2b                \n\t"\
81
82 #define YSCALEYUV2PACKEDX \
83     YSCALEYUV2PACKEDX_UV \
84     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
85
86 #define YSCALEYUV2PACKEDX_END                     \
87         :: "r" (&c->redDither),                   \
88             "m" (dummy), "m" (dummy), "m" (dummy),\
89             "r" (dest), "m" (dstW_reg), "m"(uv_off) \
90         : "%"REG_a, "%"REG_d, "%"REG_S            \
91     );
92
93 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
94     __asm__ volatile(\
95         "xor %%"REG_a", %%"REG_a"                       \n\t"\
96         ".p2align                      4                \n\t"\
97         "nop                                            \n\t"\
98         "1:                                             \n\t"\
99         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
100         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
101         "pxor                      %%mm4, %%mm4         \n\t"\
102         "pxor                      %%mm5, %%mm5         \n\t"\
103         "pxor                      %%mm6, %%mm6         \n\t"\
104         "pxor                      %%mm7, %%mm7         \n\t"\
105         ".p2align                      4                \n\t"\
106         "2:                                             \n\t"\
107         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
108         "add                          %6, %%"REG_S"      \n\t" \
109         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
110         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
111         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
112         "movq                      %%mm0, %%mm3         \n\t"\
113         "punpcklwd                 %%mm1, %%mm0         \n\t"\
114         "punpckhwd                 %%mm1, %%mm3         \n\t"\
115         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
116         "pmaddwd                   %%mm1, %%mm0         \n\t"\
117         "pmaddwd                   %%mm1, %%mm3         \n\t"\
118         "paddd                     %%mm0, %%mm4         \n\t"\
119         "paddd                     %%mm3, %%mm5         \n\t"\
120         "add                          %6, %%"REG_S"      \n\t" \
121         "movq     (%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
122         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
123         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
124         "test                  %%"REG_S", %%"REG_S"     \n\t"\
125         "movq                      %%mm2, %%mm0         \n\t"\
126         "punpcklwd                 %%mm3, %%mm2         \n\t"\
127         "punpckhwd                 %%mm3, %%mm0         \n\t"\
128         "pmaddwd                   %%mm1, %%mm2         \n\t"\
129         "pmaddwd                   %%mm1, %%mm0         \n\t"\
130         "paddd                     %%mm2, %%mm6         \n\t"\
131         "paddd                     %%mm0, %%mm7         \n\t"\
132         " jnz                         2b                \n\t"\
133         "psrad                       $16, %%mm4         \n\t"\
134         "psrad                       $16, %%mm5         \n\t"\
135         "psrad                       $16, %%mm6         \n\t"\
136         "psrad                       $16, %%mm7         \n\t"\
137         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
138         "packssdw                  %%mm5, %%mm4         \n\t"\
139         "packssdw                  %%mm7, %%mm6         \n\t"\
140         "paddw                     %%mm0, %%mm4         \n\t"\
141         "paddw                     %%mm0, %%mm6         \n\t"\
142         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
143         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
144
145 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
146     "lea                "offset"(%0), %%"REG_d"     \n\t"\
147     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
148     "pxor                      %%mm1, %%mm1         \n\t"\
149     "pxor                      %%mm5, %%mm5         \n\t"\
150     "pxor                      %%mm7, %%mm7         \n\t"\
151     "pxor                      %%mm6, %%mm6         \n\t"\
152     ".p2align                      4                \n\t"\
153     "2:                                             \n\t"\
154     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
155     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
156     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
157     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
158     "movq                      %%mm0, %%mm3         \n\t"\
159     "punpcklwd                 %%mm4, %%mm0         \n\t"\
160     "punpckhwd                 %%mm4, %%mm3         \n\t"\
161     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
162     "pmaddwd                   %%mm4, %%mm0         \n\t"\
163     "pmaddwd                   %%mm4, %%mm3         \n\t"\
164     "paddd                     %%mm0, %%mm1         \n\t"\
165     "paddd                     %%mm3, %%mm5         \n\t"\
166     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
167     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
168     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
169     "test                  %%"REG_S", %%"REG_S"     \n\t"\
170     "movq                      %%mm2, %%mm0         \n\t"\
171     "punpcklwd                 %%mm3, %%mm2         \n\t"\
172     "punpckhwd                 %%mm3, %%mm0         \n\t"\
173     "pmaddwd                   %%mm4, %%mm2         \n\t"\
174     "pmaddwd                   %%mm4, %%mm0         \n\t"\
175     "paddd                     %%mm2, %%mm7         \n\t"\
176     "paddd                     %%mm0, %%mm6         \n\t"\
177     " jnz                         2b                \n\t"\
178     "psrad                       $16, %%mm1         \n\t"\
179     "psrad                       $16, %%mm5         \n\t"\
180     "psrad                       $16, %%mm7         \n\t"\
181     "psrad                       $16, %%mm6         \n\t"\
182     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
183     "packssdw                  %%mm5, %%mm1         \n\t"\
184     "packssdw                  %%mm6, %%mm7         \n\t"\
185     "paddw                     %%mm0, %%mm1         \n\t"\
186     "paddw                     %%mm0, %%mm7         \n\t"\
187     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
188     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
189
190 #define YSCALEYUV2PACKEDX_ACCURATE \
191     YSCALEYUV2PACKEDX_ACCURATE_UV \
192     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
193
194 #define YSCALEYUV2RGBX \
195     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
196     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
197     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
198     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
199     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
200     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
201     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
202     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
203     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
204     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
205     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
206     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
207     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
208     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
209     "paddw           %%mm3, %%mm4       \n\t"\
210     "movq            %%mm2, %%mm0       \n\t"\
211     "movq            %%mm5, %%mm6       \n\t"\
212     "movq            %%mm4, %%mm3       \n\t"\
213     "punpcklwd       %%mm2, %%mm2       \n\t"\
214     "punpcklwd       %%mm5, %%mm5       \n\t"\
215     "punpcklwd       %%mm4, %%mm4       \n\t"\
216     "paddw           %%mm1, %%mm2       \n\t"\
217     "paddw           %%mm1, %%mm5       \n\t"\
218     "paddw           %%mm1, %%mm4       \n\t"\
219     "punpckhwd       %%mm0, %%mm0       \n\t"\
220     "punpckhwd       %%mm6, %%mm6       \n\t"\
221     "punpckhwd       %%mm3, %%mm3       \n\t"\
222     "paddw           %%mm7, %%mm0       \n\t"\
223     "paddw           %%mm7, %%mm6       \n\t"\
224     "paddw           %%mm7, %%mm3       \n\t"\
225     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
226     "packuswb        %%mm0, %%mm2       \n\t"\
227     "packuswb        %%mm6, %%mm5       \n\t"\
228     "packuswb        %%mm3, %%mm4       \n\t"\
229
230 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
231     "movq       "#b", "#q2"     \n\t" /* B */\
232     "movq       "#r", "#t"      \n\t" /* R */\
233     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
234     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
235     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
236     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
237     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
238     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
239     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
240     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
241     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
242     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
243 \
244     MOVNTQ(   q0,   (dst, index, 4))\
245     MOVNTQ(    b,  8(dst, index, 4))\
246     MOVNTQ(   q2, 16(dst, index, 4))\
247     MOVNTQ(   q3, 24(dst, index, 4))\
248 \
249     "add      $8, "#index"      \n\t"\
250     "cmp "#dstw", "#index"      \n\t"\
251     " jb      1b                \n\t"
252 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
253
254 static void RENAME(yuv2rgb32_X_ar)(SwsContext *c, const int16_t *lumFilter,
255                                    const int16_t **lumSrc, int lumFilterSize,
256                                    const int16_t *chrFilter, const int16_t **chrUSrc,
257                                    const int16_t **chrVSrc,
258                                    int chrFilterSize, const int16_t **alpSrc,
259                                    uint8_t *dest, int dstW, int dstY)
260 {
261     x86_reg dummy=0;
262     x86_reg dstW_reg = dstW;
263     x86_reg uv_off = c->uv_off_byte;
264
265     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
266         YSCALEYUV2PACKEDX_ACCURATE
267         YSCALEYUV2RGBX
268         "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
269         "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
270         "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
271         YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
272         "movq               "Y_TEMP"(%0), %%mm5         \n\t"
273         "psraw                        $3, %%mm1         \n\t"
274         "psraw                        $3, %%mm7         \n\t"
275         "packuswb                  %%mm7, %%mm1         \n\t"
276         WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
277         YSCALEYUV2PACKEDX_END
278     } else {
279         YSCALEYUV2PACKEDX_ACCURATE
280         YSCALEYUV2RGBX
281         "pcmpeqd %%mm7, %%mm7 \n\t"
282         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
283         YSCALEYUV2PACKEDX_END
284     }
285 }
286
287 static void RENAME(yuv2rgb32_X)(SwsContext *c, const int16_t *lumFilter,
288                                 const int16_t **lumSrc, int lumFilterSize,
289                                 const int16_t *chrFilter, const int16_t **chrUSrc,
290                                 const int16_t **chrVSrc,
291                                 int chrFilterSize, const int16_t **alpSrc,
292                                 uint8_t *dest, int dstW, int dstY)
293 {
294     x86_reg dummy=0;
295     x86_reg dstW_reg = dstW;
296     x86_reg uv_off = c->uv_off_byte;
297
298     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
299         YSCALEYUV2PACKEDX
300         YSCALEYUV2RGBX
301         YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
302         "psraw                        $3, %%mm1         \n\t"
303         "psraw                        $3, %%mm7         \n\t"
304         "packuswb                  %%mm7, %%mm1         \n\t"
305         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
306         YSCALEYUV2PACKEDX_END
307     } else {
308         YSCALEYUV2PACKEDX
309         YSCALEYUV2RGBX
310         "pcmpeqd %%mm7, %%mm7 \n\t"
311         WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
312         YSCALEYUV2PACKEDX_END
313     }
314 }
315
316 #define REAL_WRITERGB16(dst, dstw, index) \
317     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
318     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
319     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
320     "psrlq           $3, %%mm2  \n\t"\
321 \
322     "movq         %%mm2, %%mm1  \n\t"\
323     "movq         %%mm4, %%mm3  \n\t"\
324 \
325     "punpcklbw    %%mm7, %%mm3  \n\t"\
326     "punpcklbw    %%mm5, %%mm2  \n\t"\
327     "punpckhbw    %%mm7, %%mm4  \n\t"\
328     "punpckhbw    %%mm5, %%mm1  \n\t"\
329 \
330     "psllq           $3, %%mm3  \n\t"\
331     "psllq           $3, %%mm4  \n\t"\
332 \
333     "por          %%mm3, %%mm2  \n\t"\
334     "por          %%mm4, %%mm1  \n\t"\
335 \
336     MOVNTQ(%%mm2,  (dst, index, 2))\
337     MOVNTQ(%%mm1, 8(dst, index, 2))\
338 \
339     "add             $8, "#index"   \n\t"\
340     "cmp        "#dstw", "#index"   \n\t"\
341     " jb             1b             \n\t"
342 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
343
344 static void RENAME(yuv2rgb565_X_ar)(SwsContext *c, const int16_t *lumFilter,
345                                     const int16_t **lumSrc, int lumFilterSize,
346                                     const int16_t *chrFilter, const int16_t **chrUSrc,
347                                     const int16_t **chrVSrc,
348                                     int chrFilterSize, const int16_t **alpSrc,
349                                     uint8_t *dest, int dstW, int dstY)
350 {
351     x86_reg dummy=0;
352     x86_reg dstW_reg = dstW;
353     x86_reg uv_off = c->uv_off_byte;
354
355     YSCALEYUV2PACKEDX_ACCURATE
356     YSCALEYUV2RGBX
357     "pxor %%mm7, %%mm7 \n\t"
358     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
359 #ifdef DITHER1XBPP
360     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
361     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
362     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
363 #endif
364     WRITERGB16(%4, %5, %%REGa)
365     YSCALEYUV2PACKEDX_END
366 }
367
368 static void RENAME(yuv2rgb565_X)(SwsContext *c, const int16_t *lumFilter,
369                                  const int16_t **lumSrc, int lumFilterSize,
370                                  const int16_t *chrFilter, const int16_t **chrUSrc,
371                                  const int16_t **chrVSrc,
372                                  int chrFilterSize, const int16_t **alpSrc,
373                                  uint8_t *dest, int dstW, int dstY)
374 {
375     x86_reg dummy=0;
376     x86_reg dstW_reg = dstW;
377     x86_reg uv_off = c->uv_off_byte;
378
379     YSCALEYUV2PACKEDX
380     YSCALEYUV2RGBX
381     "pxor %%mm7, %%mm7 \n\t"
382     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
383 #ifdef DITHER1XBPP
384     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
385     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
386     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
387 #endif
388     WRITERGB16(%4, %5, %%REGa)
389     YSCALEYUV2PACKEDX_END
390 }
391
392 #define REAL_WRITERGB15(dst, dstw, index) \
393     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
394     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
395     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
396     "psrlq           $3, %%mm2  \n\t"\
397     "psrlq           $1, %%mm5  \n\t"\
398 \
399     "movq         %%mm2, %%mm1  \n\t"\
400     "movq         %%mm4, %%mm3  \n\t"\
401 \
402     "punpcklbw    %%mm7, %%mm3  \n\t"\
403     "punpcklbw    %%mm5, %%mm2  \n\t"\
404     "punpckhbw    %%mm7, %%mm4  \n\t"\
405     "punpckhbw    %%mm5, %%mm1  \n\t"\
406 \
407     "psllq           $2, %%mm3  \n\t"\
408     "psllq           $2, %%mm4  \n\t"\
409 \
410     "por          %%mm3, %%mm2  \n\t"\
411     "por          %%mm4, %%mm1  \n\t"\
412 \
413     MOVNTQ(%%mm2,  (dst, index, 2))\
414     MOVNTQ(%%mm1, 8(dst, index, 2))\
415 \
416     "add             $8, "#index"   \n\t"\
417     "cmp        "#dstw", "#index"   \n\t"\
418     " jb             1b             \n\t"
419 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
420
421 static void RENAME(yuv2rgb555_X_ar)(SwsContext *c, const int16_t *lumFilter,
422                                     const int16_t **lumSrc, int lumFilterSize,
423                                     const int16_t *chrFilter, const int16_t **chrUSrc,
424                                     const int16_t **chrVSrc,
425                                     int chrFilterSize, const int16_t **alpSrc,
426                                     uint8_t *dest, int dstW, int dstY)
427 {
428     x86_reg dummy=0;
429     x86_reg dstW_reg = dstW;
430     x86_reg uv_off = c->uv_off_byte;
431
432     YSCALEYUV2PACKEDX_ACCURATE
433     YSCALEYUV2RGBX
434     "pxor %%mm7, %%mm7 \n\t"
435     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
436 #ifdef DITHER1XBPP
437     "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
438     "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
439     "paddusb "RED_DITHER"(%0), %%mm5\n\t"
440 #endif
441     WRITERGB15(%4, %5, %%REGa)
442     YSCALEYUV2PACKEDX_END
443 }
444
445 static void RENAME(yuv2rgb555_X)(SwsContext *c, const int16_t *lumFilter,
446                                  const int16_t **lumSrc, int lumFilterSize,
447                                  const int16_t *chrFilter, const int16_t **chrUSrc,
448                                  const int16_t **chrVSrc,
449                                  int chrFilterSize, const int16_t **alpSrc,
450                                  uint8_t *dest, int dstW, int dstY)
451 {
452     x86_reg dummy=0;
453     x86_reg dstW_reg = dstW;
454     x86_reg uv_off = c->uv_off_byte;
455
456     YSCALEYUV2PACKEDX
457     YSCALEYUV2RGBX
458     "pxor %%mm7, %%mm7 \n\t"
459     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
460 #ifdef DITHER1XBPP
461     "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
462     "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
463     "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
464 #endif
465     WRITERGB15(%4, %5, %%REGa)
466     YSCALEYUV2PACKEDX_END
467 }
468
469 #define WRITEBGR24MMX(dst, dstw, index) \
470     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
471     "movq      %%mm2, %%mm1     \n\t" /* B */\
472     "movq      %%mm5, %%mm6     \n\t" /* R */\
473     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
474     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
475     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
476     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
477     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
478     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
479     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
480     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
481     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
482     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
483 \
484     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
485     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
486     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
487     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
488 \
489     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
490     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
491     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
492     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
493 \
494     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
495     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
496     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
497     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
498 \
499     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
500     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
501     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
502     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
503     MOVNTQ(%%mm0, (dst))\
504 \
505     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
506     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
507     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
508     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
509     MOVNTQ(%%mm6, 8(dst))\
510 \
511     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
512     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
513     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
514     MOVNTQ(%%mm5, 16(dst))\
515 \
516     "add         $24, "#dst"    \n\t"\
517 \
518     "add          $8, "#index"  \n\t"\
519     "cmp     "#dstw", "#index"  \n\t"\
520     " jb          1b            \n\t"
521
522 #define WRITEBGR24MMX2(dst, dstw, index) \
523     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
524     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
525     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
526     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
527     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
528     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
529 \
530     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
531     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
532     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
533 \
534     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
535     "por    %%mm1, %%mm6        \n\t"\
536     "por    %%mm3, %%mm6        \n\t"\
537     MOVNTQ(%%mm6, (dst))\
538 \
539     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
540     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
541     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
542     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
543 \
544     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
545     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
546     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
547 \
548     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
549     "por    %%mm3, %%mm6        \n\t"\
550     MOVNTQ(%%mm6, 8(dst))\
551 \
552     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
553     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
554     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
555 \
556     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
557     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
558     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
559 \
560     "por    %%mm1, %%mm3        \n\t"\
561     "por    %%mm3, %%mm6        \n\t"\
562     MOVNTQ(%%mm6, 16(dst))\
563 \
564     "add      $24, "#dst"       \n\t"\
565 \
566     "add       $8, "#index"     \n\t"\
567     "cmp  "#dstw", "#index"     \n\t"\
568     " jb       1b               \n\t"
569
570 #if COMPILE_TEMPLATE_MMX2
571 #undef WRITEBGR24
572 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
573 #else
574 #undef WRITEBGR24
575 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
576 #endif
577
578 static void RENAME(yuv2bgr24_X_ar)(SwsContext *c, const int16_t *lumFilter,
579                                    const int16_t **lumSrc, int lumFilterSize,
580                                    const int16_t *chrFilter, const int16_t **chrUSrc,
581                                    const int16_t **chrVSrc,
582                                    int chrFilterSize, const int16_t **alpSrc,
583                                    uint8_t *dest, int dstW, int dstY)
584 {
585     x86_reg dummy=0;
586     x86_reg dstW_reg = dstW;
587     x86_reg uv_off = c->uv_off_byte;
588
589     YSCALEYUV2PACKEDX_ACCURATE
590     YSCALEYUV2RGBX
591     "pxor %%mm7, %%mm7 \n\t"
592     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
593     "add %4, %%"REG_c"                        \n\t"
594     WRITEBGR24(%%REGc, %5, %%REGa)
595     :: "r" (&c->redDither),
596        "m" (dummy), "m" (dummy), "m" (dummy),
597        "r" (dest), "m" (dstW_reg), "m"(uv_off)
598     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
599     );
600 }
601
602 static void RENAME(yuv2bgr24_X)(SwsContext *c, const int16_t *lumFilter,
603                                 const int16_t **lumSrc, int lumFilterSize,
604                                 const int16_t *chrFilter, const int16_t **chrUSrc,
605                                 const int16_t **chrVSrc,
606                                 int chrFilterSize, const int16_t **alpSrc,
607                                 uint8_t *dest, int dstW, int dstY)
608 {
609     x86_reg dummy=0;
610     x86_reg dstW_reg = dstW;
611     x86_reg uv_off = c->uv_off_byte;
612
613     YSCALEYUV2PACKEDX
614     YSCALEYUV2RGBX
615     "pxor                    %%mm7, %%mm7       \n\t"
616     "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
617     "add                        %4, %%"REG_c"   \n\t"
618     WRITEBGR24(%%REGc, %5, %%REGa)
619     :: "r" (&c->redDither),
620        "m" (dummy), "m" (dummy), "m" (dummy),
621        "r" (dest),  "m" (dstW_reg), "m"(uv_off)
622     : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
623     );
624 }
625
626 #define REAL_WRITEYUY2(dst, dstw, index) \
627     "packuswb  %%mm3, %%mm3     \n\t"\
628     "packuswb  %%mm4, %%mm4     \n\t"\
629     "packuswb  %%mm7, %%mm1     \n\t"\
630     "punpcklbw %%mm4, %%mm3     \n\t"\
631     "movq      %%mm1, %%mm7     \n\t"\
632     "punpcklbw %%mm3, %%mm1     \n\t"\
633     "punpckhbw %%mm3, %%mm7     \n\t"\
634 \
635     MOVNTQ(%%mm1, (dst, index, 2))\
636     MOVNTQ(%%mm7, 8(dst, index, 2))\
637 \
638     "add          $8, "#index"  \n\t"\
639     "cmp     "#dstw", "#index"  \n\t"\
640     " jb          1b            \n\t"
641 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
642
643 static void RENAME(yuv2yuyv422_X_ar)(SwsContext *c, const int16_t *lumFilter,
644                                      const int16_t **lumSrc, int lumFilterSize,
645                                      const int16_t *chrFilter, const int16_t **chrUSrc,
646                                      const int16_t **chrVSrc,
647                                      int chrFilterSize, const int16_t **alpSrc,
648                                      uint8_t *dest, int dstW, int dstY)
649 {
650     x86_reg dummy=0;
651     x86_reg dstW_reg = dstW;
652     x86_reg uv_off = c->uv_off_byte;
653
654     YSCALEYUV2PACKEDX_ACCURATE
655     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
656     "psraw $3, %%mm3    \n\t"
657     "psraw $3, %%mm4    \n\t"
658     "psraw $3, %%mm1    \n\t"
659     "psraw $3, %%mm7    \n\t"
660     WRITEYUY2(%4, %5, %%REGa)
661     YSCALEYUV2PACKEDX_END
662 }
663
664 static void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter,
665                                   const int16_t **lumSrc, int lumFilterSize,
666                                   const int16_t *chrFilter, const int16_t **chrUSrc,
667                                   const int16_t **chrVSrc,
668                                   int chrFilterSize, const int16_t **alpSrc,
669                                   uint8_t *dest, int dstW, int dstY)
670 {
671     x86_reg dummy=0;
672     x86_reg dstW_reg = dstW;
673     x86_reg uv_off = c->uv_off_byte;
674
675     YSCALEYUV2PACKEDX
676     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
677     "psraw $3, %%mm3    \n\t"
678     "psraw $3, %%mm4    \n\t"
679     "psraw $3, %%mm1    \n\t"
680     "psraw $3, %%mm7    \n\t"
681     WRITEYUY2(%4, %5, %%REGa)
682     YSCALEYUV2PACKEDX_END
683 }
684
685 #define REAL_YSCALEYUV2RGB_UV(index, c) \
686     "xor            "#index", "#index"  \n\t"\
687     ".p2align              4            \n\t"\
688     "1:                                 \n\t"\
689     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
690     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
691     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
692     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
693     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
694     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
695     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
696     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
697     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
698     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
699     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
700     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
701     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
702     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
703     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
704     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
705     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
706     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
707     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
708     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
709     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
710     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
711
712 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
713     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
714     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
715     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
716     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
717     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
718     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
719     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
720     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
721     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
722     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
723     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
724     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
725
726 #define REAL_YSCALEYUV2RGB_COEFF(c) \
727     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
728     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
729     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
730     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
731     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
732     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
733     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
734     "paddw             %%mm3, %%mm4     \n\t"\
735     "movq              %%mm2, %%mm0     \n\t"\
736     "movq              %%mm5, %%mm6     \n\t"\
737     "movq              %%mm4, %%mm3     \n\t"\
738     "punpcklwd         %%mm2, %%mm2     \n\t"\
739     "punpcklwd         %%mm5, %%mm5     \n\t"\
740     "punpcklwd         %%mm4, %%mm4     \n\t"\
741     "paddw             %%mm1, %%mm2     \n\t"\
742     "paddw             %%mm1, %%mm5     \n\t"\
743     "paddw             %%mm1, %%mm4     \n\t"\
744     "punpckhwd         %%mm0, %%mm0     \n\t"\
745     "punpckhwd         %%mm6, %%mm6     \n\t"\
746     "punpckhwd         %%mm3, %%mm3     \n\t"\
747     "paddw             %%mm7, %%mm0     \n\t"\
748     "paddw             %%mm7, %%mm6     \n\t"\
749     "paddw             %%mm7, %%mm3     \n\t"\
750     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
751     "packuswb          %%mm0, %%mm2     \n\t"\
752     "packuswb          %%mm6, %%mm5     \n\t"\
753     "packuswb          %%mm3, %%mm4     \n\t"\
754
755 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
756
757 #define YSCALEYUV2RGB(index, c) \
758     REAL_YSCALEYUV2RGB_UV(index, c) \
759     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
760     REAL_YSCALEYUV2RGB_COEFF(c)
761
762 /**
763  * vertical bilinear scale YV12 to RGB
764  */
765 static void RENAME(yuv2rgb32_2)(SwsContext *c, const int16_t *buf[2],
766                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
767                                 const int16_t *abuf[2], uint8_t *dest,
768                                 int dstW, int yalpha, int uvalpha, int y)
769 {
770     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
771                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
772
773     if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
774         const int16_t *abuf0 = abuf[0], *abuf1 = abuf[1];
775 #if ARCH_X86_64
776         __asm__ volatile(
777             YSCALEYUV2RGB(%%r8, %5)
778             YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
779             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
780             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
781             "packuswb            %%mm7, %%mm1       \n\t"
782             WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
783             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "r" (dest),
784                "a" (&c->redDither),
785                "r" (abuf0), "r" (abuf1)
786             : "%r8"
787         );
788 #else
789         *(const uint16_t **)(&c->u_temp)=abuf0;
790         *(const uint16_t **)(&c->v_temp)=abuf1;
791         __asm__ volatile(
792             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
793             "mov        %4, %%"REG_b"               \n\t"
794             "push %%"REG_BP"                        \n\t"
795             YSCALEYUV2RGB(%%REGBP, %5)
796             "push                   %0              \n\t"
797             "push                   %1              \n\t"
798             "mov          "U_TEMP"(%5), %0          \n\t"
799             "mov          "V_TEMP"(%5), %1          \n\t"
800             YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
801             "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
802             "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
803             "packuswb            %%mm7, %%mm1       \n\t"
804             "pop                    %1              \n\t"
805             "pop                    %0              \n\t"
806             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
807             "pop %%"REG_BP"                         \n\t"
808             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
809             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
810                "a" (&c->redDither)
811         );
812 #endif
813     } else {
814         __asm__ volatile(
815             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
816             "mov        %4, %%"REG_b"               \n\t"
817             "push %%"REG_BP"                        \n\t"
818             YSCALEYUV2RGB(%%REGBP, %5)
819             "pcmpeqd %%mm7, %%mm7                   \n\t"
820             WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
821             "pop %%"REG_BP"                         \n\t"
822             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
823             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
824                "a" (&c->redDither)
825         );
826     }
827 }
828
829 static void RENAME(yuv2bgr24_2)(SwsContext *c, const int16_t *buf[2],
830                                 const int16_t *ubuf[2], const int16_t *vbuf[2],
831                                 const int16_t *abuf[2], uint8_t *dest,
832                                 int dstW, int yalpha, int uvalpha, int y)
833 {
834     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
835                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
836
837     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
838     __asm__ volatile(
839         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
840         "mov        %4, %%"REG_b"               \n\t"
841         "push %%"REG_BP"                        \n\t"
842         YSCALEYUV2RGB(%%REGBP, %5)
843         "pxor    %%mm7, %%mm7                   \n\t"
844         WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
845         "pop %%"REG_BP"                         \n\t"
846         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
847         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
848            "a" (&c->redDither)
849     );
850 }
851
852 static void RENAME(yuv2rgb555_2)(SwsContext *c, const int16_t *buf[2],
853                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
854                                  const int16_t *abuf[2], uint8_t *dest,
855                                  int dstW, int yalpha, int uvalpha, int y)
856 {
857     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
858                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
859
860     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
861     __asm__ volatile(
862         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
863         "mov        %4, %%"REG_b"               \n\t"
864         "push %%"REG_BP"                        \n\t"
865         YSCALEYUV2RGB(%%REGBP, %5)
866         "pxor    %%mm7, %%mm7                   \n\t"
867         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
868 #ifdef DITHER1XBPP
869         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
870         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
871         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
872 #endif
873         WRITERGB15(%%REGb, 8280(%5), %%REGBP)
874         "pop %%"REG_BP"                         \n\t"
875         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
876         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
877            "a" (&c->redDither)
878     );
879 }
880
881 static void RENAME(yuv2rgb565_2)(SwsContext *c, const int16_t *buf[2],
882                                  const int16_t *ubuf[2], const int16_t *vbuf[2],
883                                  const int16_t *abuf[2], uint8_t *dest,
884                                  int dstW, int yalpha, int uvalpha, int y)
885 {
886     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
887                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
888
889     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
890     __asm__ volatile(
891         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
892         "mov        %4, %%"REG_b"               \n\t"
893         "push %%"REG_BP"                        \n\t"
894         YSCALEYUV2RGB(%%REGBP, %5)
895         "pxor    %%mm7, %%mm7                   \n\t"
896         /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
897 #ifdef DITHER1XBPP
898         "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
899         "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
900         "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
901 #endif
902         WRITERGB16(%%REGb, 8280(%5), %%REGBP)
903         "pop %%"REG_BP"                         \n\t"
904         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
905         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
906            "a" (&c->redDither)
907     );
908 }
909
910 #define REAL_YSCALEYUV2PACKED(index, c) \
911     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
912     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
913     "psraw                $3, %%mm0                           \n\t"\
914     "psraw                $3, %%mm1                           \n\t"\
915     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
916     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
917     "xor            "#index", "#index"                        \n\t"\
918     ".p2align              4            \n\t"\
919     "1:                                 \n\t"\
920     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
921     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
922     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
923     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
924     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
925     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
926     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
927     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
928     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
929     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
930     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
931     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
932     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
933     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
934     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
935     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
936     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
937     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
938     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
939     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
940     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
941     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
942     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
943     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
944     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
945     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
946     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
947
948 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
949
950 static void RENAME(yuv2yuyv422_2)(SwsContext *c, const int16_t *buf[2],
951                                   const int16_t *ubuf[2], const int16_t *vbuf[2],
952                                   const int16_t *abuf[2], uint8_t *dest,
953                                   int dstW, int yalpha, int uvalpha, int y)
954 {
955     const int16_t *buf0  = buf[0],  *buf1  = buf[1],
956                   *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
957
958     //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
959     __asm__ volatile(
960         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
961         "mov %4, %%"REG_b"                        \n\t"
962         "push %%"REG_BP"                        \n\t"
963         YSCALEYUV2PACKED(%%REGBP, %5)
964         WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
965         "pop %%"REG_BP"                         \n\t"
966         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
967         :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
968            "a" (&c->redDither)
969     );
970 }
971
972 #define REAL_YSCALEYUV2RGB1(index, c) \
973     "xor            "#index", "#index"  \n\t"\
974     ".p2align              4            \n\t"\
975     "1:                                 \n\t"\
976     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
977     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
978     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
979     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
980     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
981     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
982     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
983     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
984     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
985     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
986     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
987     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
988     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
989     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
990     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
991     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
992     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
993     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
994     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
995     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
996     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
997     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
998     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
999     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1000     "paddw             %%mm3, %%mm4     \n\t"\
1001     "movq              %%mm2, %%mm0     \n\t"\
1002     "movq              %%mm5, %%mm6     \n\t"\
1003     "movq              %%mm4, %%mm3     \n\t"\
1004     "punpcklwd         %%mm2, %%mm2     \n\t"\
1005     "punpcklwd         %%mm5, %%mm5     \n\t"\
1006     "punpcklwd         %%mm4, %%mm4     \n\t"\
1007     "paddw             %%mm1, %%mm2     \n\t"\
1008     "paddw             %%mm1, %%mm5     \n\t"\
1009     "paddw             %%mm1, %%mm4     \n\t"\
1010     "punpckhwd         %%mm0, %%mm0     \n\t"\
1011     "punpckhwd         %%mm6, %%mm6     \n\t"\
1012     "punpckhwd         %%mm3, %%mm3     \n\t"\
1013     "paddw             %%mm7, %%mm0     \n\t"\
1014     "paddw             %%mm7, %%mm6     \n\t"\
1015     "paddw             %%mm7, %%mm3     \n\t"\
1016     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1017     "packuswb          %%mm0, %%mm2     \n\t"\
1018     "packuswb          %%mm6, %%mm5     \n\t"\
1019     "packuswb          %%mm3, %%mm4     \n\t"\
1020
1021 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
1022
1023 // do vertical chrominance interpolation
1024 #define REAL_YSCALEYUV2RGB1b(index, c) \
1025     "xor            "#index", "#index"  \n\t"\
1026     ".p2align              4            \n\t"\
1027     "1:                                 \n\t"\
1028     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1029     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1030     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1031     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1032     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1033     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1034     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1035     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1036     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
1037     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
1038     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
1039     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
1040     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
1041     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
1042     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
1043     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
1044     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
1045     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1046     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1047     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1048     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
1049     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
1050     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
1051     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
1052     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
1053     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
1054     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
1055     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
1056     "paddw             %%mm3, %%mm4     \n\t"\
1057     "movq              %%mm2, %%mm0     \n\t"\
1058     "movq              %%mm5, %%mm6     \n\t"\
1059     "movq              %%mm4, %%mm3     \n\t"\
1060     "punpcklwd         %%mm2, %%mm2     \n\t"\
1061     "punpcklwd         %%mm5, %%mm5     \n\t"\
1062     "punpcklwd         %%mm4, %%mm4     \n\t"\
1063     "paddw             %%mm1, %%mm2     \n\t"\
1064     "paddw             %%mm1, %%mm5     \n\t"\
1065     "paddw             %%mm1, %%mm4     \n\t"\
1066     "punpckhwd         %%mm0, %%mm0     \n\t"\
1067     "punpckhwd         %%mm6, %%mm6     \n\t"\
1068     "punpckhwd         %%mm3, %%mm3     \n\t"\
1069     "paddw             %%mm7, %%mm0     \n\t"\
1070     "paddw             %%mm7, %%mm6     \n\t"\
1071     "paddw             %%mm7, %%mm3     \n\t"\
1072     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
1073     "packuswb          %%mm0, %%mm2     \n\t"\
1074     "packuswb          %%mm6, %%mm5     \n\t"\
1075     "packuswb          %%mm3, %%mm4     \n\t"\
1076
1077 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
1078
1079 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
1080     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
1081     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
1082     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
1083     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
1084     "packuswb          %%mm1, %%mm7     \n\t"
1085 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
1086
1087 /**
1088  * YV12 to RGB without scaling or interpolating
1089  */
1090 static void RENAME(yuv2rgb32_1)(SwsContext *c, const int16_t *buf0,
1091                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1092                                 const int16_t *abuf0, uint8_t *dest,
1093                                 int dstW, int uvalpha, int y)
1094 {
1095     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1096     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1097
1098     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1099         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1100             __asm__ volatile(
1101                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1102                 "mov        %4, %%"REG_b"               \n\t"
1103                 "push %%"REG_BP"                        \n\t"
1104                 YSCALEYUV2RGB1(%%REGBP, %5)
1105                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1106                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1107                 "pop %%"REG_BP"                         \n\t"
1108                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1109                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1110                    "a" (&c->redDither)
1111             );
1112         } else {
1113             __asm__ volatile(
1114                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1115                 "mov        %4, %%"REG_b"               \n\t"
1116                 "push %%"REG_BP"                        \n\t"
1117                 YSCALEYUV2RGB1(%%REGBP, %5)
1118                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1119                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1120                 "pop %%"REG_BP"                         \n\t"
1121                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1122                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1123                    "a" (&c->redDither)
1124             );
1125         }
1126     } else {
1127         if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1128             __asm__ volatile(
1129                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1130                 "mov        %4, %%"REG_b"               \n\t"
1131                 "push %%"REG_BP"                        \n\t"
1132                 YSCALEYUV2RGB1b(%%REGBP, %5)
1133                 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1134                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1135                 "pop %%"REG_BP"                         \n\t"
1136                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1137                 :: "c" (buf0), "d" (abuf0), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1138                    "a" (&c->redDither)
1139             );
1140         } else {
1141             __asm__ volatile(
1142                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1143                 "mov        %4, %%"REG_b"               \n\t"
1144                 "push %%"REG_BP"                        \n\t"
1145                 YSCALEYUV2RGB1b(%%REGBP, %5)
1146                 "pcmpeqd %%mm7, %%mm7                   \n\t"
1147                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1148                 "pop %%"REG_BP"                         \n\t"
1149                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1150                 :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1151                    "a" (&c->redDither)
1152             );
1153         }
1154     }
1155 }
1156
1157 static void RENAME(yuv2bgr24_1)(SwsContext *c, const int16_t *buf0,
1158                                 const int16_t *ubuf[2], const int16_t *bguf[2],
1159                                 const int16_t *abuf0, uint8_t *dest,
1160                                 int dstW, int uvalpha, int y)
1161 {
1162     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1163     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1164
1165     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1166         __asm__ volatile(
1167             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1168             "mov        %4, %%"REG_b"               \n\t"
1169             "push %%"REG_BP"                        \n\t"
1170             YSCALEYUV2RGB1(%%REGBP, %5)
1171             "pxor    %%mm7, %%mm7                   \n\t"
1172             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1173             "pop %%"REG_BP"                         \n\t"
1174             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1175             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1176                "a" (&c->redDither)
1177         );
1178     } else {
1179         __asm__ volatile(
1180             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1181             "mov        %4, %%"REG_b"               \n\t"
1182             "push %%"REG_BP"                        \n\t"
1183             YSCALEYUV2RGB1b(%%REGBP, %5)
1184             "pxor    %%mm7, %%mm7                   \n\t"
1185             WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1186             "pop %%"REG_BP"                         \n\t"
1187             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1188             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1189                "a" (&c->redDither)
1190         );
1191     }
1192 }
1193
1194 static void RENAME(yuv2rgb555_1)(SwsContext *c, const int16_t *buf0,
1195                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1196                                  const int16_t *abuf0, uint8_t *dest,
1197                                  int dstW, int uvalpha, int y)
1198 {
1199     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1200     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1201
1202     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1203         __asm__ volatile(
1204             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1205             "mov        %4, %%"REG_b"               \n\t"
1206             "push %%"REG_BP"                        \n\t"
1207             YSCALEYUV2RGB1(%%REGBP, %5)
1208             "pxor    %%mm7, %%mm7                   \n\t"
1209             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1210 #ifdef DITHER1XBPP
1211             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1212             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1213             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1214 #endif
1215             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1216             "pop %%"REG_BP"                         \n\t"
1217             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1218             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1219                "a" (&c->redDither)
1220         );
1221     } else {
1222         __asm__ volatile(
1223             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1224             "mov        %4, %%"REG_b"               \n\t"
1225             "push %%"REG_BP"                        \n\t"
1226             YSCALEYUV2RGB1b(%%REGBP, %5)
1227             "pxor    %%mm7, %%mm7                   \n\t"
1228             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1229 #ifdef DITHER1XBPP
1230             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1231             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1232             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1233 #endif
1234             WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1235             "pop %%"REG_BP"                         \n\t"
1236             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1237             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1238                "a" (&c->redDither)
1239         );
1240     }
1241 }
1242
1243 static void RENAME(yuv2rgb565_1)(SwsContext *c, const int16_t *buf0,
1244                                  const int16_t *ubuf[2], const int16_t *bguf[2],
1245                                  const int16_t *abuf0, uint8_t *dest,
1246                                  int dstW, int uvalpha, int y)
1247 {
1248     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1249     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1250
1251     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1252         __asm__ volatile(
1253             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1254             "mov        %4, %%"REG_b"               \n\t"
1255             "push %%"REG_BP"                        \n\t"
1256             YSCALEYUV2RGB1(%%REGBP, %5)
1257             "pxor    %%mm7, %%mm7                   \n\t"
1258             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1259 #ifdef DITHER1XBPP
1260             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1261             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1262             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1263 #endif
1264             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1265             "pop %%"REG_BP"                         \n\t"
1266             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1267             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1268                "a" (&c->redDither)
1269         );
1270     } else {
1271         __asm__ volatile(
1272             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1273             "mov        %4, %%"REG_b"               \n\t"
1274             "push %%"REG_BP"                        \n\t"
1275             YSCALEYUV2RGB1b(%%REGBP, %5)
1276             "pxor    %%mm7, %%mm7                   \n\t"
1277             /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1278 #ifdef DITHER1XBPP
1279             "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1280             "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1281             "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1282 #endif
1283             WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1284             "pop %%"REG_BP"                         \n\t"
1285             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1286             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1287                "a" (&c->redDither)
1288         );
1289     }
1290 }
1291
1292 #define REAL_YSCALEYUV2PACKED1(index, c) \
1293     "xor            "#index", "#index"  \n\t"\
1294     ".p2align              4            \n\t"\
1295     "1:                                 \n\t"\
1296     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
1297     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1298     "movq     (%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
1299     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1300     "psraw                $7, %%mm3     \n\t" \
1301     "psraw                $7, %%mm4     \n\t" \
1302     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1303     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1304     "psraw                $7, %%mm1     \n\t" \
1305     "psraw                $7, %%mm7     \n\t" \
1306
1307 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
1308
1309 #define REAL_YSCALEYUV2PACKED1b(index, c) \
1310     "xor "#index", "#index"             \n\t"\
1311     ".p2align              4            \n\t"\
1312     "1:                                 \n\t"\
1313     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
1314     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
1315     "add   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1316     "movq     (%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
1317     "movq     (%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
1318     "sub   "UV_OFF_PX"("#c"), "#index"  \n\t" \
1319     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
1320     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
1321     "psrlw                $8, %%mm3     \n\t" \
1322     "psrlw                $8, %%mm4     \n\t" \
1323     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
1324     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
1325     "psraw                $7, %%mm1     \n\t" \
1326     "psraw                $7, %%mm7     \n\t"
1327 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
1328
1329 static void RENAME(yuv2yuyv422_1)(SwsContext *c, const int16_t *buf0,
1330                                   const int16_t *ubuf[2], const int16_t *bguf[2],
1331                                   const int16_t *abuf0, uint8_t *dest,
1332                                   int dstW, int uvalpha, int y)
1333 {
1334     const int16_t *ubuf0 = ubuf[0], *ubuf1 = ubuf[1];
1335     const int16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1336
1337     if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1338         __asm__ volatile(
1339             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1340             "mov        %4, %%"REG_b"               \n\t"
1341             "push %%"REG_BP"                        \n\t"
1342             YSCALEYUV2PACKED1(%%REGBP, %5)
1343             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1344             "pop %%"REG_BP"                         \n\t"
1345             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1346             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1347                "a" (&c->redDither)
1348         );
1349     } else {
1350         __asm__ volatile(
1351             "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1352             "mov        %4, %%"REG_b"               \n\t"
1353             "push %%"REG_BP"                        \n\t"
1354             YSCALEYUV2PACKED1b(%%REGBP, %5)
1355             WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1356             "pop %%"REG_BP"                         \n\t"
1357             "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1358             :: "c" (buf0), "d" (buf1), "S" (ubuf0), "D" (ubuf1), "m" (dest),
1359                "a" (&c->redDither)
1360         );
1361     }
1362 }
1363
1364 #if !COMPILE_TEMPLATE_MMX2
1365 //FIXME yuy2* can read up to 7 samples too much
1366
1367 static void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src,
1368                             int width, uint32_t *unused)
1369 {
1370     __asm__ volatile(
1371         "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1372         "mov                    %0, %%"REG_a"       \n\t"
1373         "1:                                         \n\t"
1374         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1375         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1376         "pand                %%mm2, %%mm0           \n\t"
1377         "pand                %%mm2, %%mm1           \n\t"
1378         "packuswb            %%mm1, %%mm0           \n\t"
1379         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1380         "add                    $8, %%"REG_a"       \n\t"
1381         " js                    1b                  \n\t"
1382         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1383         : "%"REG_a
1384     );
1385 }
1386
1387 static void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV,
1388                              const uint8_t *src1, const uint8_t *src2,
1389                              int width, uint32_t *unused)
1390 {
1391     __asm__ volatile(
1392         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1393         "mov                    %0, %%"REG_a"       \n\t"
1394         "1:                                         \n\t"
1395         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1396         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1397         "psrlw                  $8, %%mm0           \n\t"
1398         "psrlw                  $8, %%mm1           \n\t"
1399         "packuswb            %%mm1, %%mm0           \n\t"
1400         "movq                %%mm0, %%mm1           \n\t"
1401         "psrlw                  $8, %%mm0           \n\t"
1402         "pand                %%mm4, %%mm1           \n\t"
1403         "packuswb            %%mm0, %%mm0           \n\t"
1404         "packuswb            %%mm1, %%mm1           \n\t"
1405         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1406         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1407         "add                    $4, %%"REG_a"       \n\t"
1408         " js                    1b                  \n\t"
1409         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1410         : "%"REG_a
1411     );
1412     assert(src1 == src2);
1413 }
1414
1415 /* This is almost identical to the previous, end exists only because
1416  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1417 static void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src,
1418                             int width, uint32_t *unused)
1419 {
1420     __asm__ volatile(
1421         "mov                  %0, %%"REG_a"         \n\t"
1422         "1:                                         \n\t"
1423         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1424         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1425         "psrlw                $8, %%mm0             \n\t"
1426         "psrlw                $8, %%mm1             \n\t"
1427         "packuswb          %%mm1, %%mm0             \n\t"
1428         "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1429         "add                  $8, %%"REG_a"         \n\t"
1430         " js                  1b                    \n\t"
1431         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1432         : "%"REG_a
1433     );
1434 }
1435
1436 static void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV,
1437                              const uint8_t *src1, const uint8_t *src2,
1438                              int width, uint32_t *unused)
1439 {
1440     __asm__ volatile(
1441         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1442         "mov                    %0, %%"REG_a"       \n\t"
1443         "1:                                         \n\t"
1444         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1445         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1446         "pand                %%mm4, %%mm0           \n\t"
1447         "pand                %%mm4, %%mm1           \n\t"
1448         "packuswb            %%mm1, %%mm0           \n\t"
1449         "movq                %%mm0, %%mm1           \n\t"
1450         "psrlw                  $8, %%mm0           \n\t"
1451         "pand                %%mm4, %%mm1           \n\t"
1452         "packuswb            %%mm0, %%mm0           \n\t"
1453         "packuswb            %%mm1, %%mm1           \n\t"
1454         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1455         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1456         "add                    $4, %%"REG_a"       \n\t"
1457         " js                    1b                  \n\t"
1458         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1459         : "%"REG_a
1460     );
1461     assert(src1 == src2);
1462 }
1463
1464 static av_always_inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1465                                               const uint8_t *src, int width)
1466 {
1467     __asm__ volatile(
1468         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1469         "mov                    %0, %%"REG_a"       \n\t"
1470         "1:                                         \n\t"
1471         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1472         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1473         "movq                %%mm0, %%mm2           \n\t"
1474         "movq                %%mm1, %%mm3           \n\t"
1475         "pand                %%mm4, %%mm0           \n\t"
1476         "pand                %%mm4, %%mm1           \n\t"
1477         "psrlw                  $8, %%mm2           \n\t"
1478         "psrlw                  $8, %%mm3           \n\t"
1479         "packuswb            %%mm1, %%mm0           \n\t"
1480         "packuswb            %%mm3, %%mm2           \n\t"
1481         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1482         "movq                %%mm2, (%3, %%"REG_a") \n\t"
1483         "add                    $8, %%"REG_a"       \n\t"
1484         " js                    1b                  \n\t"
1485         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1486         : "%"REG_a
1487     );
1488 }
1489
1490 static void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1491                              const uint8_t *src1, const uint8_t *src2,
1492                              int width, uint32_t *unused)
1493 {
1494     RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1495 }
1496
1497 static void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1498                              const uint8_t *src1, const uint8_t *src2,
1499                              int width, uint32_t *unused)
1500 {
1501     RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1502 }
1503 #endif /* !COMPILE_TEMPLATE_MMX2 */
1504
1505 static av_always_inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, const uint8_t *src,
1506                                                   int width, enum PixelFormat srcFormat)
1507 {
1508
1509     if(srcFormat == PIX_FMT_BGR24) {
1510         __asm__ volatile(
1511             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1512             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1513             :
1514         );
1515     } else {
1516         __asm__ volatile(
1517             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1518             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1519             :
1520         );
1521     }
1522
1523     __asm__ volatile(
1524         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1525         "mov                        %2, %%"REG_a"   \n\t"
1526         "pxor                    %%mm7, %%mm7       \n\t"
1527         "1:                                         \n\t"
1528         PREFETCH"               64(%0)              \n\t"
1529         "movd                     (%0), %%mm0       \n\t"
1530         "movd                    2(%0), %%mm1       \n\t"
1531         "movd                    6(%0), %%mm2       \n\t"
1532         "movd                    8(%0), %%mm3       \n\t"
1533         "add                       $12, %0          \n\t"
1534         "punpcklbw               %%mm7, %%mm0       \n\t"
1535         "punpcklbw               %%mm7, %%mm1       \n\t"
1536         "punpcklbw               %%mm7, %%mm2       \n\t"
1537         "punpcklbw               %%mm7, %%mm3       \n\t"
1538         "pmaddwd                 %%mm5, %%mm0       \n\t"
1539         "pmaddwd                 %%mm6, %%mm1       \n\t"
1540         "pmaddwd                 %%mm5, %%mm2       \n\t"
1541         "pmaddwd                 %%mm6, %%mm3       \n\t"
1542         "paddd                   %%mm1, %%mm0       \n\t"
1543         "paddd                   %%mm3, %%mm2       \n\t"
1544         "paddd                   %%mm4, %%mm0       \n\t"
1545         "paddd                   %%mm4, %%mm2       \n\t"
1546         "psrad                     $15, %%mm0       \n\t"
1547         "psrad                     $15, %%mm2       \n\t"
1548         "packssdw                %%mm2, %%mm0       \n\t"
1549         "packuswb                %%mm0, %%mm0       \n\t"
1550         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1551         "add                        $4, %%"REG_a"   \n\t"
1552         " js                        1b              \n\t"
1553     : "+r" (src)
1554     : "r" (dst+width), "g" ((x86_reg)-width)
1555     : "%"REG_a
1556     );
1557 }
1558
1559 static void RENAME(bgr24ToY)(uint8_t *dst, const uint8_t *src,
1560                              int width, uint32_t *unused)
1561 {
1562     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1563 }
1564
1565 static void RENAME(rgb24ToY)(uint8_t *dst, const uint8_t *src,
1566                              int width, uint32_t *unused)
1567 {
1568     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1569 }
1570
1571 static av_always_inline void RENAME(bgr24ToUV_mmx)(uint8_t *dstU, uint8_t *dstV,
1572                                                    const uint8_t *src, int width,
1573                                                    enum PixelFormat srcFormat)
1574 {
1575     __asm__ volatile(
1576         "movq                    24(%4), %%mm6       \n\t"
1577         "mov                        %3, %%"REG_a"   \n\t"
1578         "pxor                    %%mm7, %%mm7       \n\t"
1579         "1:                                         \n\t"
1580         PREFETCH"               64(%0)              \n\t"
1581         "movd                     (%0), %%mm0       \n\t"
1582         "movd                    2(%0), %%mm1       \n\t"
1583         "punpcklbw               %%mm7, %%mm0       \n\t"
1584         "punpcklbw               %%mm7, %%mm1       \n\t"
1585         "movq                    %%mm0, %%mm2       \n\t"
1586         "movq                    %%mm1, %%mm3       \n\t"
1587         "pmaddwd                  (%4), %%mm0       \n\t"
1588         "pmaddwd                 8(%4), %%mm1       \n\t"
1589         "pmaddwd                16(%4), %%mm2       \n\t"
1590         "pmaddwd                 %%mm6, %%mm3       \n\t"
1591         "paddd                   %%mm1, %%mm0       \n\t"
1592         "paddd                   %%mm3, %%mm2       \n\t"
1593
1594         "movd                    6(%0), %%mm1       \n\t"
1595         "movd                    8(%0), %%mm3       \n\t"
1596         "add                       $12, %0          \n\t"
1597         "punpcklbw               %%mm7, %%mm1       \n\t"
1598         "punpcklbw               %%mm7, %%mm3       \n\t"
1599         "movq                    %%mm1, %%mm4       \n\t"
1600         "movq                    %%mm3, %%mm5       \n\t"
1601         "pmaddwd                  (%4), %%mm1       \n\t"
1602         "pmaddwd                 8(%4), %%mm3       \n\t"
1603         "pmaddwd                16(%4), %%mm4       \n\t"
1604         "pmaddwd                 %%mm6, %%mm5       \n\t"
1605         "paddd                   %%mm3, %%mm1       \n\t"
1606         "paddd                   %%mm5, %%mm4       \n\t"
1607
1608         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1609         "paddd                   %%mm3, %%mm0       \n\t"
1610         "paddd                   %%mm3, %%mm2       \n\t"
1611         "paddd                   %%mm3, %%mm1       \n\t"
1612         "paddd                   %%mm3, %%mm4       \n\t"
1613         "psrad                     $15, %%mm0       \n\t"
1614         "psrad                     $15, %%mm2       \n\t"
1615         "psrad                     $15, %%mm1       \n\t"
1616         "psrad                     $15, %%mm4       \n\t"
1617         "packssdw                %%mm1, %%mm0       \n\t"
1618         "packssdw                %%mm4, %%mm2       \n\t"
1619         "packuswb                %%mm0, %%mm0       \n\t"
1620         "packuswb                %%mm2, %%mm2       \n\t"
1621         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1622         "movd                %%mm2, (%2, %%"REG_a") \n\t"
1623         "add                        $4, %%"REG_a"   \n\t"
1624         " js                        1b              \n\t"
1625     : "+r" (src)
1626     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1627     : "%"REG_a
1628     );
1629 }
1630
1631 static void RENAME(bgr24ToUV)(uint8_t *dstU, uint8_t *dstV,
1632                               const uint8_t *src1, const uint8_t *src2,
1633                               int width, uint32_t *unused)
1634 {
1635     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1636     assert(src1 == src2);
1637 }
1638
1639 static void RENAME(rgb24ToUV)(uint8_t *dstU, uint8_t *dstV,
1640                               const uint8_t *src1, const uint8_t *src2,
1641                               int width, uint32_t *unused)
1642 {
1643     assert(src1==src2);
1644     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1645 }
1646
1647 #if COMPILE_TEMPLATE_MMX2
1648 static void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
1649                                  int dstWidth, const uint8_t *src,
1650                                  int srcW, int xInc)
1651 {
1652     int16_t *filterPos = c->hLumFilterPos;
1653     int16_t *filter    = c->hLumFilter;
1654     void    *mmx2FilterCode= c->lumMmx2FilterCode;
1655     int i;
1656 #if defined(PIC)
1657     DECLARE_ALIGNED(8, uint64_t, ebxsave);
1658 #endif
1659
1660     __asm__ volatile(
1661 #if defined(PIC)
1662         "mov               %%"REG_b", %5        \n\t"
1663 #endif
1664         "pxor                  %%mm7, %%mm7     \n\t"
1665         "mov                      %0, %%"REG_c" \n\t"
1666         "mov                      %1, %%"REG_D" \n\t"
1667         "mov                      %2, %%"REG_d" \n\t"
1668         "mov                      %3, %%"REG_b" \n\t"
1669         "xor               %%"REG_a", %%"REG_a" \n\t" // i
1670         PREFETCH"        (%%"REG_c")            \n\t"
1671         PREFETCH"      32(%%"REG_c")            \n\t"
1672         PREFETCH"      64(%%"REG_c")            \n\t"
1673
1674 #if ARCH_X86_64
1675 #define CALL_MMX2_FILTER_CODE \
1676         "movl            (%%"REG_b"), %%esi     \n\t"\
1677         "call                    *%4            \n\t"\
1678         "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
1679         "add               %%"REG_S", %%"REG_c" \n\t"\
1680         "add               %%"REG_a", %%"REG_D" \n\t"\
1681         "xor               %%"REG_a", %%"REG_a" \n\t"\
1682
1683 #else
1684 #define CALL_MMX2_FILTER_CODE \
1685         "movl (%%"REG_b"), %%esi        \n\t"\
1686         "call         *%4                       \n\t"\
1687         "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
1688         "add               %%"REG_a", %%"REG_D" \n\t"\
1689         "xor               %%"REG_a", %%"REG_a" \n\t"\
1690
1691 #endif /* ARCH_X86_64 */
1692
1693         CALL_MMX2_FILTER_CODE
1694         CALL_MMX2_FILTER_CODE
1695         CALL_MMX2_FILTER_CODE
1696         CALL_MMX2_FILTER_CODE
1697         CALL_MMX2_FILTER_CODE
1698         CALL_MMX2_FILTER_CODE
1699         CALL_MMX2_FILTER_CODE
1700         CALL_MMX2_FILTER_CODE
1701
1702 #if defined(PIC)
1703         "mov                      %5, %%"REG_b" \n\t"
1704 #endif
1705         :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
1706            "m" (mmx2FilterCode)
1707 #if defined(PIC)
1708           ,"m" (ebxsave)
1709 #endif
1710         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
1711 #if !defined(PIC)
1712          ,"%"REG_b
1713 #endif
1714     );
1715
1716     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
1717         dst[i] = src[srcW-1]*128;
1718 }
1719
1720 static void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2,
1721                                  int dstWidth, const uint8_t *src1,
1722                                  const uint8_t *src2, int srcW, int xInc)
1723 {
1724     int16_t *filterPos = c->hChrFilterPos;
1725     int16_t *filter    = c->hChrFilter;
1726     void    *mmx2FilterCode= c->chrMmx2FilterCode;
1727     int i;
1728 #if defined(PIC)
1729     DECLARE_ALIGNED(8, uint64_t, ebxsave);
1730 #endif
1731
1732     __asm__ volatile(
1733 #if defined(PIC)
1734         "mov          %%"REG_b", %7         \n\t"
1735 #endif
1736         "pxor             %%mm7, %%mm7      \n\t"
1737         "mov                 %0, %%"REG_c"  \n\t"
1738         "mov                 %1, %%"REG_D"  \n\t"
1739         "mov                 %2, %%"REG_d"  \n\t"
1740         "mov                 %3, %%"REG_b"  \n\t"
1741         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
1742         PREFETCH"   (%%"REG_c")             \n\t"
1743         PREFETCH" 32(%%"REG_c")             \n\t"
1744         PREFETCH" 64(%%"REG_c")             \n\t"
1745
1746         CALL_MMX2_FILTER_CODE
1747         CALL_MMX2_FILTER_CODE
1748         CALL_MMX2_FILTER_CODE
1749         CALL_MMX2_FILTER_CODE
1750         "xor          %%"REG_a", %%"REG_a"  \n\t" // i
1751         "mov                 %5, %%"REG_c"  \n\t" // src
1752         "mov                 %6, %%"REG_D"  \n\t" // buf2
1753         PREFETCH"   (%%"REG_c")             \n\t"
1754         PREFETCH" 32(%%"REG_c")             \n\t"
1755         PREFETCH" 64(%%"REG_c")             \n\t"
1756
1757         CALL_MMX2_FILTER_CODE
1758         CALL_MMX2_FILTER_CODE
1759         CALL_MMX2_FILTER_CODE
1760         CALL_MMX2_FILTER_CODE
1761
1762 #if defined(PIC)
1763         "mov %7, %%"REG_b"    \n\t"
1764 #endif
1765         :: "m" (src1), "m" (dst1), "m" (filter), "m" (filterPos),
1766            "m" (mmx2FilterCode), "m" (src2), "m"(dst2)
1767 #if defined(PIC)
1768           ,"m" (ebxsave)
1769 #endif
1770         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
1771 #if !defined(PIC)
1772          ,"%"REG_b
1773 #endif
1774     );
1775
1776     for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
1777         dst1[i] = src1[srcW-1]*128;
1778         dst2[i] = src2[srcW-1]*128;
1779     }
1780 }
1781 #endif /* COMPILE_TEMPLATE_MMX2 */
1782
1783 static av_cold void RENAME(sws_init_swScale)(SwsContext *c)
1784 {
1785     enum PixelFormat srcFormat = c->srcFormat,
1786                      dstFormat = c->dstFormat;
1787
1788     if (!is16BPS(dstFormat) && !is9_OR_10BPS(dstFormat) &&
1789         dstFormat != PIX_FMT_NV12 && dstFormat != PIX_FMT_NV21) {
1790         if (!(c->flags & SWS_BITEXACT)) {
1791             if (c->flags & SWS_ACCURATE_RND) {
1792                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1793                     switch (c->dstFormat) {
1794                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X_ar);   break;
1795                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X_ar);   break;
1796                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X_ar);  break;
1797                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X_ar);  break;
1798                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X_ar); break;
1799                     default: break;
1800                     }
1801                 }
1802             } else {
1803                 if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1804                     switch (c->dstFormat) {
1805                     case PIX_FMT_RGB32:   c->yuv2packedX = RENAME(yuv2rgb32_X);   break;
1806                     case PIX_FMT_BGR24:   c->yuv2packedX = RENAME(yuv2bgr24_X);   break;
1807                     case PIX_FMT_RGB555:  c->yuv2packedX = RENAME(yuv2rgb555_X);  break;
1808                     case PIX_FMT_RGB565:  c->yuv2packedX = RENAME(yuv2rgb565_X);  break;
1809                     case PIX_FMT_YUYV422: c->yuv2packedX = RENAME(yuv2yuyv422_X); break;
1810                     default: break;
1811                     }
1812                 }
1813             }
1814         }
1815         if (!(c->flags & SWS_FULL_CHR_H_INT)) {
1816             switch (c->dstFormat) {
1817             case PIX_FMT_RGB32:
1818                 c->yuv2packed1 = RENAME(yuv2rgb32_1);
1819                 c->yuv2packed2 = RENAME(yuv2rgb32_2);
1820                 break;
1821             case PIX_FMT_BGR24:
1822                 c->yuv2packed1 = RENAME(yuv2bgr24_1);
1823                 c->yuv2packed2 = RENAME(yuv2bgr24_2);
1824                 break;
1825             case PIX_FMT_RGB555:
1826                 c->yuv2packed1 = RENAME(yuv2rgb555_1);
1827                 c->yuv2packed2 = RENAME(yuv2rgb555_2);
1828                 break;
1829             case PIX_FMT_RGB565:
1830                 c->yuv2packed1 = RENAME(yuv2rgb565_1);
1831                 c->yuv2packed2 = RENAME(yuv2rgb565_2);
1832                 break;
1833             case PIX_FMT_YUYV422:
1834                 c->yuv2packed1 = RENAME(yuv2yuyv422_1);
1835                 c->yuv2packed2 = RENAME(yuv2yuyv422_2);
1836                 break;
1837             default:
1838                 break;
1839             }
1840         }
1841     }
1842
1843     if (c->srcBpc == 8 && c->dstBpc <= 10) {
1844     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
1845 #if COMPILE_TEMPLATE_MMX2
1846     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
1847     {
1848         c->hyscale_fast = RENAME(hyscale_fast);
1849         c->hcscale_fast = RENAME(hcscale_fast);
1850     } else {
1851 #endif /* COMPILE_TEMPLATE_MMX2 */
1852         c->hyscale_fast = NULL;
1853         c->hcscale_fast = NULL;
1854 #if COMPILE_TEMPLATE_MMX2
1855     }
1856 #endif /* COMPILE_TEMPLATE_MMX2 */
1857     }
1858
1859 #if !COMPILE_TEMPLATE_MMX2
1860     switch(srcFormat) {
1861         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;
1862         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;
1863         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;
1864         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;
1865         default: break;
1866     }
1867 #endif /* !COMPILE_TEMPLATE_MMX2 */
1868     if (!c->chrSrcHSubSample) {
1869         switch(srcFormat) {
1870         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
1871         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
1872         default: break;
1873         }
1874     }
1875
1876     switch (srcFormat) {
1877 #if !COMPILE_TEMPLATE_MMX2
1878     case PIX_FMT_YUYV422  :
1879     case PIX_FMT_Y400A    : c->lumToYV12 = RENAME(yuy2ToY); break;
1880     case PIX_FMT_UYVY422  : c->lumToYV12 = RENAME(uyvyToY); break;
1881 #endif /* !COMPILE_TEMPLATE_MMX2 */
1882     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
1883     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
1884     default: break;
1885     }
1886 #if !COMPILE_TEMPLATE_MMX2
1887     if (c->alpPixBuf) {
1888         switch (srcFormat) {
1889         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;
1890         default: break;
1891         }
1892     }
1893 #endif /* !COMPILE_TEMPLATE_MMX2 */
1894 }