swscale: further cleanup swscale.c
[ffmpeg.git] / libswscale / x86 / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of Libav.
5  *
6  * Libav is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * Libav is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with Libav; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 DECLARE_ASM_CONST(8, uint64_t, bF8)=       0xF8F8F8F8F8F8F8F8LL;
22 DECLARE_ASM_CONST(8, uint64_t, bFC)=       0xFCFCFCFCFCFCFCFCLL;
23 DECLARE_ASM_CONST(8, uint64_t, w10)=       0x0010001000100010LL;
24 DECLARE_ASM_CONST(8, uint64_t, w02)=       0x0002000200020002LL;
25 DECLARE_ASM_CONST(8, uint64_t, bm00001111)=0x00000000FFFFFFFFLL;
26 DECLARE_ASM_CONST(8, uint64_t, bm00000111)=0x0000000000FFFFFFLL;
27 DECLARE_ASM_CONST(8, uint64_t, bm11111000)=0xFFFFFFFFFF000000LL;
28 DECLARE_ASM_CONST(8, uint64_t, bm01010101)=0x00FF00FF00FF00FFLL;
29
30 const DECLARE_ALIGNED(8, uint64_t, ff_dither4)[2] = {
31         0x0103010301030103LL,
32         0x0200020002000200LL,};
33
34 const DECLARE_ALIGNED(8, uint64_t, ff_dither8)[2] = {
35         0x0602060206020602LL,
36         0x0004000400040004LL,};
37
38 DECLARE_ASM_CONST(8, uint64_t, b16Mask)=   0x001F001F001F001FLL;
39 DECLARE_ASM_CONST(8, uint64_t, g16Mask)=   0x07E007E007E007E0LL;
40 DECLARE_ASM_CONST(8, uint64_t, r16Mask)=   0xF800F800F800F800LL;
41 DECLARE_ASM_CONST(8, uint64_t, b15Mask)=   0x001F001F001F001FLL;
42 DECLARE_ASM_CONST(8, uint64_t, g15Mask)=   0x03E003E003E003E0LL;
43 DECLARE_ASM_CONST(8, uint64_t, r15Mask)=   0x7C007C007C007C00LL;
44
45 DECLARE_ALIGNED(8, const uint64_t, ff_M24A)         = 0x00FF0000FF0000FFLL;
46 DECLARE_ALIGNED(8, const uint64_t, ff_M24B)         = 0xFF0000FF0000FF00LL;
47 DECLARE_ALIGNED(8, const uint64_t, ff_M24C)         = 0x0000FF0000FF0000LL;
48
49 #ifdef FAST_BGR2YV12
50 DECLARE_ALIGNED(8, const uint64_t, ff_bgr2YCoeff)   = 0x000000210041000DULL;
51 DECLARE_ALIGNED(8, const uint64_t, ff_bgr2UCoeff)   = 0x0000FFEEFFDC0038ULL;
52 DECLARE_ALIGNED(8, const uint64_t, ff_bgr2VCoeff)   = 0x00000038FFD2FFF8ULL;
53 #else
54 DECLARE_ALIGNED(8, const uint64_t, ff_bgr2YCoeff)   = 0x000020E540830C8BULL;
55 DECLARE_ALIGNED(8, const uint64_t, ff_bgr2UCoeff)   = 0x0000ED0FDAC23831ULL;
56 DECLARE_ALIGNED(8, const uint64_t, ff_bgr2VCoeff)   = 0x00003831D0E6F6EAULL;
57 #endif /* FAST_BGR2YV12 */
58 DECLARE_ALIGNED(8, const uint64_t, ff_bgr2YOffset)  = 0x1010101010101010ULL;
59 DECLARE_ALIGNED(8, const uint64_t, ff_bgr2UVOffset) = 0x8080808080808080ULL;
60 DECLARE_ALIGNED(8, const uint64_t, ff_w1111)        = 0x0001000100010001ULL;
61
62 DECLARE_ASM_CONST(8, uint64_t, ff_bgr24toY1Coeff) = 0x0C88000040870C88ULL;
63 DECLARE_ASM_CONST(8, uint64_t, ff_bgr24toY2Coeff) = 0x20DE4087000020DEULL;
64 DECLARE_ASM_CONST(8, uint64_t, ff_rgb24toY1Coeff) = 0x20DE0000408720DEULL;
65 DECLARE_ASM_CONST(8, uint64_t, ff_rgb24toY2Coeff) = 0x0C88408700000C88ULL;
66 DECLARE_ASM_CONST(8, uint64_t, ff_bgr24toYOffset) = 0x0008400000084000ULL;
67
68 DECLARE_ASM_CONST(8, uint64_t, ff_bgr24toUV)[2][4] = {
69     {0x38380000DAC83838ULL, 0xECFFDAC80000ECFFULL, 0xF6E40000D0E3F6E4ULL, 0x3838D0E300003838ULL},
70     {0xECFF0000DAC8ECFFULL, 0x3838DAC800003838ULL, 0x38380000D0E33838ULL, 0xF6E4D0E30000F6E4ULL},
71 };
72
73 DECLARE_ASM_CONST(8, uint64_t, ff_bgr24toUVOffset)= 0x0040400000404000ULL;
74
75 #undef REAL_MOVNTQ
76 #undef MOVNTQ
77 #undef PAVGB
78 #undef PREFETCH
79
80 #if COMPILE_TEMPLATE_AMD3DNOW
81 #define PREFETCH  "prefetch"
82 #elif COMPILE_TEMPLATE_MMX2
83 #define PREFETCH "prefetchnta"
84 #else
85 #define PREFETCH  " # nop"
86 #endif
87
88 #if COMPILE_TEMPLATE_MMX2
89 #define PAVGB(a,b) "pavgb " #a ", " #b " \n\t"
90 #elif COMPILE_TEMPLATE_AMD3DNOW
91 #define PAVGB(a,b) "pavgusb " #a ", " #b " \n\t"
92 #endif
93
94 #if COMPILE_TEMPLATE_MMX2
95 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
96 #else
97 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
98 #endif
99 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
100
101 #define YSCALEYUV2YV12X(x, offset, dest, width) \
102     __asm__ volatile(\
103         "xor                          %%"REG_a", %%"REG_a"  \n\t"\
104         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
105         "movq                             %%mm3, %%mm4      \n\t"\
106         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
107         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
108         ".p2align                             4             \n\t" /* FIXME Unroll? */\
109         "1:                                                 \n\t"\
110         "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\
111         "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm2      \n\t" /* srcData */\
112         "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm5      \n\t" /* srcData */\
113         "add                                $16, %%"REG_d"  \n\t"\
114         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
115         "test                         %%"REG_S", %%"REG_S"  \n\t"\
116         "pmulhw                           %%mm0, %%mm2      \n\t"\
117         "pmulhw                           %%mm0, %%mm5      \n\t"\
118         "paddw                            %%mm2, %%mm3      \n\t"\
119         "paddw                            %%mm5, %%mm4      \n\t"\
120         " jnz                                1b             \n\t"\
121         "psraw                               $3, %%mm3      \n\t"\
122         "psraw                               $3, %%mm4      \n\t"\
123         "packuswb                         %%mm4, %%mm3      \n\t"\
124         MOVNTQ(%%mm3, (%1, %%REGa))\
125         "add                                 $8, %%"REG_a"  \n\t"\
126         "cmp                                 %2, %%"REG_a"  \n\t"\
127         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
128         "movq                             %%mm3, %%mm4      \n\t"\
129         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
130         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
131         "jb                                  1b             \n\t"\
132         :: "r" (&c->redDither),\
133         "r" (dest), "g" ((x86_reg)width)\
134         : "%"REG_a, "%"REG_d, "%"REG_S\
135     );
136
137 #define YSCALEYUV2YV12X_ACCURATE(x, offset, dest, width) \
138     __asm__ volatile(\
139         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
140         "xor                          %%"REG_a", %%"REG_a"  \n\t"\
141         "pxor                             %%mm4, %%mm4      \n\t"\
142         "pxor                             %%mm5, %%mm5      \n\t"\
143         "pxor                             %%mm6, %%mm6      \n\t"\
144         "pxor                             %%mm7, %%mm7      \n\t"\
145         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
146         ".p2align                             4             \n\t"\
147         "1:                                                 \n\t"\
148         "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm0      \n\t" /* srcData */\
149         "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm2      \n\t" /* srcData */\
150         "mov        "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"  \n\t"\
151         "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm1      \n\t" /* srcData */\
152         "movq                             %%mm0, %%mm3      \n\t"\
153         "punpcklwd                        %%mm1, %%mm0      \n\t"\
154         "punpckhwd                        %%mm1, %%mm3      \n\t"\
155         "movq       "STR(APCK_COEF)"(%%"REG_d"), %%mm1      \n\t" /* filterCoeff */\
156         "pmaddwd                          %%mm1, %%mm0      \n\t"\
157         "pmaddwd                          %%mm1, %%mm3      \n\t"\
158         "paddd                            %%mm0, %%mm4      \n\t"\
159         "paddd                            %%mm3, %%mm5      \n\t"\
160         "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm3      \n\t" /* srcData */\
161         "mov        "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"  \n\t"\
162         "add                  $"STR(APCK_SIZE)", %%"REG_d"  \n\t"\
163         "test                         %%"REG_S", %%"REG_S"  \n\t"\
164         "movq                             %%mm2, %%mm0      \n\t"\
165         "punpcklwd                        %%mm3, %%mm2      \n\t"\
166         "punpckhwd                        %%mm3, %%mm0      \n\t"\
167         "pmaddwd                          %%mm1, %%mm2      \n\t"\
168         "pmaddwd                          %%mm1, %%mm0      \n\t"\
169         "paddd                            %%mm2, %%mm6      \n\t"\
170         "paddd                            %%mm0, %%mm7      \n\t"\
171         " jnz                                1b             \n\t"\
172         "psrad                              $16, %%mm4      \n\t"\
173         "psrad                              $16, %%mm5      \n\t"\
174         "psrad                              $16, %%mm6      \n\t"\
175         "psrad                              $16, %%mm7      \n\t"\
176         "movq             "VROUNDER_OFFSET"(%0), %%mm0      \n\t"\
177         "packssdw                         %%mm5, %%mm4      \n\t"\
178         "packssdw                         %%mm7, %%mm6      \n\t"\
179         "paddw                            %%mm0, %%mm4      \n\t"\
180         "paddw                            %%mm0, %%mm6      \n\t"\
181         "psraw                               $3, %%mm4      \n\t"\
182         "psraw                               $3, %%mm6      \n\t"\
183         "packuswb                         %%mm6, %%mm4      \n\t"\
184         MOVNTQ(%%mm4, (%1, %%REGa))\
185         "add                                 $8, %%"REG_a"  \n\t"\
186         "cmp                                 %2, %%"REG_a"  \n\t"\
187         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
188         "pxor                             %%mm4, %%mm4      \n\t"\
189         "pxor                             %%mm5, %%mm5      \n\t"\
190         "pxor                             %%mm6, %%mm6      \n\t"\
191         "pxor                             %%mm7, %%mm7      \n\t"\
192         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
193         "jb                                  1b             \n\t"\
194         :: "r" (&c->redDither),\
195         "r" (dest), "g" ((x86_reg)width)\
196         : "%"REG_a, "%"REG_d, "%"REG_S\
197     );
198
199 #define YSCALEYUV2YV121 \
200     "mov %2, %%"REG_a"                    \n\t"\
201     ".p2align               4             \n\t" /* FIXME Unroll? */\
202     "1:                                   \n\t"\
203     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
204     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
205     "psraw                 $7, %%mm0      \n\t"\
206     "psraw                 $7, %%mm1      \n\t"\
207     "packuswb           %%mm1, %%mm0      \n\t"\
208     MOVNTQ(%%mm0, (%1, %%REGa))\
209     "add                   $8, %%"REG_a"  \n\t"\
210     "jnc                   1b             \n\t"
211
212 #define YSCALEYUV2YV121_ACCURATE \
213     "mov %2, %%"REG_a"                    \n\t"\
214     "pcmpeqw %%mm7, %%mm7                 \n\t"\
215     "psrlw                 $15, %%mm7     \n\t"\
216     "psllw                  $6, %%mm7     \n\t"\
217     ".p2align                4            \n\t" /* FIXME Unroll? */\
218     "1:                                   \n\t"\
219     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
220     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
221     "paddsw             %%mm7, %%mm0      \n\t"\
222     "paddsw             %%mm7, %%mm1      \n\t"\
223     "psraw                 $7, %%mm0      \n\t"\
224     "psraw                 $7, %%mm1      \n\t"\
225     "packuswb           %%mm1, %%mm0      \n\t"\
226     MOVNTQ(%%mm0, (%1, %%REGa))\
227     "add                   $8, %%"REG_a"  \n\t"\
228     "jnc                   1b             \n\t"
229
230 /*
231     :: "m" (-lumFilterSize), "m" (-chrFilterSize),
232        "m" (lumMmxFilter+lumFilterSize*4), "m" (chrMmxFilter+chrFilterSize*4),
233        "r" (dest), "m" (dstW_reg),
234        "m" (lumSrc+lumFilterSize), "m" (chrSrc+chrFilterSize)
235     : "%eax", "%ebx", "%ecx", "%edx", "%esi"
236 */
237 #define YSCALEYUV2PACKEDX_UV \
238     __asm__ volatile(\
239         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
240         ".p2align                      4                \n\t"\
241         "nop                                            \n\t"\
242         "1:                                             \n\t"\
243         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
244         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
245         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
246         "movq                      %%mm3, %%mm4         \n\t"\
247         ".p2align                      4                \n\t"\
248         "2:                                             \n\t"\
249         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
250         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
251         "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
252         "add                         $16, %%"REG_d"     \n\t"\
253         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
254         "pmulhw                    %%mm0, %%mm2         \n\t"\
255         "pmulhw                    %%mm0, %%mm5         \n\t"\
256         "paddw                     %%mm2, %%mm3         \n\t"\
257         "paddw                     %%mm5, %%mm4         \n\t"\
258         "test                  %%"REG_S", %%"REG_S"     \n\t"\
259         " jnz                         2b                \n\t"\
260
261 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
262     "lea                "offset"(%0), %%"REG_d"     \n\t"\
263     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
264     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
265     "movq                    "#dst1", "#dst2"       \n\t"\
266     ".p2align                      4                \n\t"\
267     "2:                                             \n\t"\
268     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
269     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
270     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
271     "add                         $16, %%"REG_d"            \n\t"\
272     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
273     "pmulhw                 "#coeff", "#src1"       \n\t"\
274     "pmulhw                 "#coeff", "#src2"       \n\t"\
275     "paddw                   "#src1", "#dst1"       \n\t"\
276     "paddw                   "#src2", "#dst2"       \n\t"\
277     "test                  %%"REG_S", %%"REG_S"     \n\t"\
278     " jnz                         2b                \n\t"\
279
280 #define YSCALEYUV2PACKEDX \
281     YSCALEYUV2PACKEDX_UV \
282     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
283
284 #define YSCALEYUV2PACKEDX_END                     \
285         :: "r" (&c->redDither),                   \
286             "m" (dummy), "m" (dummy), "m" (dummy),\
287             "r" (dest), "m" (dstW_reg)            \
288         : "%"REG_a, "%"REG_d, "%"REG_S            \
289     );
290
291 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
292     __asm__ volatile(\
293         "xor %%"REG_a", %%"REG_a"                       \n\t"\
294         ".p2align                      4                \n\t"\
295         "nop                                            \n\t"\
296         "1:                                             \n\t"\
297         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
298         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
299         "pxor                      %%mm4, %%mm4         \n\t"\
300         "pxor                      %%mm5, %%mm5         \n\t"\
301         "pxor                      %%mm6, %%mm6         \n\t"\
302         "pxor                      %%mm7, %%mm7         \n\t"\
303         ".p2align                      4                \n\t"\
304         "2:                                             \n\t"\
305         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
306         "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
307         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
308         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
309         "movq                      %%mm0, %%mm3         \n\t"\
310         "punpcklwd                 %%mm1, %%mm0         \n\t"\
311         "punpckhwd                 %%mm1, %%mm3         \n\t"\
312         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
313         "pmaddwd                   %%mm1, %%mm0         \n\t"\
314         "pmaddwd                   %%mm1, %%mm3         \n\t"\
315         "paddd                     %%mm0, %%mm4         \n\t"\
316         "paddd                     %%mm3, %%mm5         \n\t"\
317         "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
318         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
319         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
320         "test                  %%"REG_S", %%"REG_S"     \n\t"\
321         "movq                      %%mm2, %%mm0         \n\t"\
322         "punpcklwd                 %%mm3, %%mm2         \n\t"\
323         "punpckhwd                 %%mm3, %%mm0         \n\t"\
324         "pmaddwd                   %%mm1, %%mm2         \n\t"\
325         "pmaddwd                   %%mm1, %%mm0         \n\t"\
326         "paddd                     %%mm2, %%mm6         \n\t"\
327         "paddd                     %%mm0, %%mm7         \n\t"\
328         " jnz                         2b                \n\t"\
329         "psrad                       $16, %%mm4         \n\t"\
330         "psrad                       $16, %%mm5         \n\t"\
331         "psrad                       $16, %%mm6         \n\t"\
332         "psrad                       $16, %%mm7         \n\t"\
333         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
334         "packssdw                  %%mm5, %%mm4         \n\t"\
335         "packssdw                  %%mm7, %%mm6         \n\t"\
336         "paddw                     %%mm0, %%mm4         \n\t"\
337         "paddw                     %%mm0, %%mm6         \n\t"\
338         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
339         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
340
341 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
342     "lea                "offset"(%0), %%"REG_d"     \n\t"\
343     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
344     "pxor                      %%mm1, %%mm1         \n\t"\
345     "pxor                      %%mm5, %%mm5         \n\t"\
346     "pxor                      %%mm7, %%mm7         \n\t"\
347     "pxor                      %%mm6, %%mm6         \n\t"\
348     ".p2align                      4                \n\t"\
349     "2:                                             \n\t"\
350     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
351     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
352     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
353     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
354     "movq                      %%mm0, %%mm3         \n\t"\
355     "punpcklwd                 %%mm4, %%mm0         \n\t"\
356     "punpckhwd                 %%mm4, %%mm3         \n\t"\
357     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
358     "pmaddwd                   %%mm4, %%mm0         \n\t"\
359     "pmaddwd                   %%mm4, %%mm3         \n\t"\
360     "paddd                     %%mm0, %%mm1         \n\t"\
361     "paddd                     %%mm3, %%mm5         \n\t"\
362     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
363     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
364     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
365     "test                  %%"REG_S", %%"REG_S"     \n\t"\
366     "movq                      %%mm2, %%mm0         \n\t"\
367     "punpcklwd                 %%mm3, %%mm2         \n\t"\
368     "punpckhwd                 %%mm3, %%mm0         \n\t"\
369     "pmaddwd                   %%mm4, %%mm2         \n\t"\
370     "pmaddwd                   %%mm4, %%mm0         \n\t"\
371     "paddd                     %%mm2, %%mm7         \n\t"\
372     "paddd                     %%mm0, %%mm6         \n\t"\
373     " jnz                         2b                \n\t"\
374     "psrad                       $16, %%mm1         \n\t"\
375     "psrad                       $16, %%mm5         \n\t"\
376     "psrad                       $16, %%mm7         \n\t"\
377     "psrad                       $16, %%mm6         \n\t"\
378     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
379     "packssdw                  %%mm5, %%mm1         \n\t"\
380     "packssdw                  %%mm6, %%mm7         \n\t"\
381     "paddw                     %%mm0, %%mm1         \n\t"\
382     "paddw                     %%mm0, %%mm7         \n\t"\
383     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
384     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
385
386 #define YSCALEYUV2PACKEDX_ACCURATE \
387     YSCALEYUV2PACKEDX_ACCURATE_UV \
388     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
389
390 #define YSCALEYUV2RGBX \
391     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
392     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
393     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
394     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
395     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
396     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
397     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
398     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
399     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
400     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
401     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
402     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
403     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
404     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
405     "paddw           %%mm3, %%mm4       \n\t"\
406     "movq            %%mm2, %%mm0       \n\t"\
407     "movq            %%mm5, %%mm6       \n\t"\
408     "movq            %%mm4, %%mm3       \n\t"\
409     "punpcklwd       %%mm2, %%mm2       \n\t"\
410     "punpcklwd       %%mm5, %%mm5       \n\t"\
411     "punpcklwd       %%mm4, %%mm4       \n\t"\
412     "paddw           %%mm1, %%mm2       \n\t"\
413     "paddw           %%mm1, %%mm5       \n\t"\
414     "paddw           %%mm1, %%mm4       \n\t"\
415     "punpckhwd       %%mm0, %%mm0       \n\t"\
416     "punpckhwd       %%mm6, %%mm6       \n\t"\
417     "punpckhwd       %%mm3, %%mm3       \n\t"\
418     "paddw           %%mm7, %%mm0       \n\t"\
419     "paddw           %%mm7, %%mm6       \n\t"\
420     "paddw           %%mm7, %%mm3       \n\t"\
421     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
422     "packuswb        %%mm0, %%mm2       \n\t"\
423     "packuswb        %%mm6, %%mm5       \n\t"\
424     "packuswb        %%mm3, %%mm4       \n\t"\
425
426 #define REAL_YSCALEYUV2PACKED(index, c) \
427     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
428     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
429     "psraw                $3, %%mm0                           \n\t"\
430     "psraw                $3, %%mm1                           \n\t"\
431     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
432     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
433     "xor            "#index", "#index"                        \n\t"\
434     ".p2align              4            \n\t"\
435     "1:                                 \n\t"\
436     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
437     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
438     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
439     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
440     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
441     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
442     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
443     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
444     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
445     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
446     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
447     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
448     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
449     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
450     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
451     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
452     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
453     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
454     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
455     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
456     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
457     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
458     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
459     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
460     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
461
462 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
463
464 #define REAL_YSCALEYUV2RGB_UV(index, c) \
465     "xor            "#index", "#index"  \n\t"\
466     ".p2align              4            \n\t"\
467     "1:                                 \n\t"\
468     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
469     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
470     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
471     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
472     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
473     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
474     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
475     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
476     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
477     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
478     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
479     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
480     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
481     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
482     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
483     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
484     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
485     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
486     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
487     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
488
489 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
490     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
491     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
492     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
493     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
494     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
495     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
496     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
497     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
498     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
499     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
500     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
501     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
502
503 #define REAL_YSCALEYUV2RGB_COEFF(c) \
504     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
505     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
506     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
507     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
508     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
509     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
510     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
511     "paddw             %%mm3, %%mm4     \n\t"\
512     "movq              %%mm2, %%mm0     \n\t"\
513     "movq              %%mm5, %%mm6     \n\t"\
514     "movq              %%mm4, %%mm3     \n\t"\
515     "punpcklwd         %%mm2, %%mm2     \n\t"\
516     "punpcklwd         %%mm5, %%mm5     \n\t"\
517     "punpcklwd         %%mm4, %%mm4     \n\t"\
518     "paddw             %%mm1, %%mm2     \n\t"\
519     "paddw             %%mm1, %%mm5     \n\t"\
520     "paddw             %%mm1, %%mm4     \n\t"\
521     "punpckhwd         %%mm0, %%mm0     \n\t"\
522     "punpckhwd         %%mm6, %%mm6     \n\t"\
523     "punpckhwd         %%mm3, %%mm3     \n\t"\
524     "paddw             %%mm7, %%mm0     \n\t"\
525     "paddw             %%mm7, %%mm6     \n\t"\
526     "paddw             %%mm7, %%mm3     \n\t"\
527     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
528     "packuswb          %%mm0, %%mm2     \n\t"\
529     "packuswb          %%mm6, %%mm5     \n\t"\
530     "packuswb          %%mm3, %%mm4     \n\t"\
531
532 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
533
534 #define YSCALEYUV2RGB(index, c) \
535     REAL_YSCALEYUV2RGB_UV(index, c) \
536     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
537     REAL_YSCALEYUV2RGB_COEFF(c)
538
539 #define REAL_YSCALEYUV2PACKED1(index, c) \
540     "xor            "#index", "#index"  \n\t"\
541     ".p2align              4            \n\t"\
542     "1:                                 \n\t"\
543     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
544     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
545     "psraw                $7, %%mm3     \n\t" \
546     "psraw                $7, %%mm4     \n\t" \
547     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
548     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
549     "psraw                $7, %%mm1     \n\t" \
550     "psraw                $7, %%mm7     \n\t" \
551
552 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
553
554 #define REAL_YSCALEYUV2RGB1(index, c) \
555     "xor            "#index", "#index"  \n\t"\
556     ".p2align              4            \n\t"\
557     "1:                                 \n\t"\
558     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
559     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
560     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
561     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
562     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
563     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
564     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
565     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
566     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
567     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
568     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
569     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
570     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
571     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
572     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
573     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
574     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
575     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
576     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
577     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
578     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
579     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
580     "paddw             %%mm3, %%mm4     \n\t"\
581     "movq              %%mm2, %%mm0     \n\t"\
582     "movq              %%mm5, %%mm6     \n\t"\
583     "movq              %%mm4, %%mm3     \n\t"\
584     "punpcklwd         %%mm2, %%mm2     \n\t"\
585     "punpcklwd         %%mm5, %%mm5     \n\t"\
586     "punpcklwd         %%mm4, %%mm4     \n\t"\
587     "paddw             %%mm1, %%mm2     \n\t"\
588     "paddw             %%mm1, %%mm5     \n\t"\
589     "paddw             %%mm1, %%mm4     \n\t"\
590     "punpckhwd         %%mm0, %%mm0     \n\t"\
591     "punpckhwd         %%mm6, %%mm6     \n\t"\
592     "punpckhwd         %%mm3, %%mm3     \n\t"\
593     "paddw             %%mm7, %%mm0     \n\t"\
594     "paddw             %%mm7, %%mm6     \n\t"\
595     "paddw             %%mm7, %%mm3     \n\t"\
596     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
597     "packuswb          %%mm0, %%mm2     \n\t"\
598     "packuswb          %%mm6, %%mm5     \n\t"\
599     "packuswb          %%mm3, %%mm4     \n\t"\
600
601 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
602
603 #define REAL_YSCALEYUV2PACKED1b(index, c) \
604     "xor "#index", "#index"             \n\t"\
605     ".p2align              4            \n\t"\
606     "1:                                 \n\t"\
607     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
608     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
609     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
610     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
611     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
612     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
613     "psrlw                $8, %%mm3     \n\t" \
614     "psrlw                $8, %%mm4     \n\t" \
615     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
616     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
617     "psraw                $7, %%mm1     \n\t" \
618     "psraw                $7, %%mm7     \n\t"
619 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
620
621 // do vertical chrominance interpolation
622 #define REAL_YSCALEYUV2RGB1b(index, c) \
623     "xor            "#index", "#index"  \n\t"\
624     ".p2align              4            \n\t"\
625     "1:                                 \n\t"\
626     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
627     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
628     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
629     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
630     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
631     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
632     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
633     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
634     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
635     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
636     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
637     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
638     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
639     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
640     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
641     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
642     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
643     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
644     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
645     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
646     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
647     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
648     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
649     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
650     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
651     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
652     "paddw             %%mm3, %%mm4     \n\t"\
653     "movq              %%mm2, %%mm0     \n\t"\
654     "movq              %%mm5, %%mm6     \n\t"\
655     "movq              %%mm4, %%mm3     \n\t"\
656     "punpcklwd         %%mm2, %%mm2     \n\t"\
657     "punpcklwd         %%mm5, %%mm5     \n\t"\
658     "punpcklwd         %%mm4, %%mm4     \n\t"\
659     "paddw             %%mm1, %%mm2     \n\t"\
660     "paddw             %%mm1, %%mm5     \n\t"\
661     "paddw             %%mm1, %%mm4     \n\t"\
662     "punpckhwd         %%mm0, %%mm0     \n\t"\
663     "punpckhwd         %%mm6, %%mm6     \n\t"\
664     "punpckhwd         %%mm3, %%mm3     \n\t"\
665     "paddw             %%mm7, %%mm0     \n\t"\
666     "paddw             %%mm7, %%mm6     \n\t"\
667     "paddw             %%mm7, %%mm3     \n\t"\
668     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
669     "packuswb          %%mm0, %%mm2     \n\t"\
670     "packuswb          %%mm6, %%mm5     \n\t"\
671     "packuswb          %%mm3, %%mm4     \n\t"\
672
673 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
674
675 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
676     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
677     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
678     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
679     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
680     "packuswb          %%mm1, %%mm7     \n\t"
681 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
682
683 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
684     "movq       "#b", "#q2"     \n\t" /* B */\
685     "movq       "#r", "#t"      \n\t" /* R */\
686     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
687     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
688     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
689     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
690     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
691     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
692     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
693     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
694     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
695     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
696 \
697     MOVNTQ(   q0,   (dst, index, 4))\
698     MOVNTQ(    b,  8(dst, index, 4))\
699     MOVNTQ(   q2, 16(dst, index, 4))\
700     MOVNTQ(   q3, 24(dst, index, 4))\
701 \
702     "add      $8, "#index"      \n\t"\
703     "cmp "#dstw", "#index"      \n\t"\
704     " jb      1b                \n\t"
705 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
706
707 #define REAL_WRITERGB16(dst, dstw, index) \
708     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
709     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
710     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
711     "psrlq           $3, %%mm2  \n\t"\
712 \
713     "movq         %%mm2, %%mm1  \n\t"\
714     "movq         %%mm4, %%mm3  \n\t"\
715 \
716     "punpcklbw    %%mm7, %%mm3  \n\t"\
717     "punpcklbw    %%mm5, %%mm2  \n\t"\
718     "punpckhbw    %%mm7, %%mm4  \n\t"\
719     "punpckhbw    %%mm5, %%mm1  \n\t"\
720 \
721     "psllq           $3, %%mm3  \n\t"\
722     "psllq           $3, %%mm4  \n\t"\
723 \
724     "por          %%mm3, %%mm2  \n\t"\
725     "por          %%mm4, %%mm1  \n\t"\
726 \
727     MOVNTQ(%%mm2,  (dst, index, 2))\
728     MOVNTQ(%%mm1, 8(dst, index, 2))\
729 \
730     "add             $8, "#index"   \n\t"\
731     "cmp        "#dstw", "#index"   \n\t"\
732     " jb             1b             \n\t"
733 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
734
735 #define REAL_WRITERGB15(dst, dstw, index) \
736     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
737     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
738     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
739     "psrlq           $3, %%mm2  \n\t"\
740     "psrlq           $1, %%mm5  \n\t"\
741 \
742     "movq         %%mm2, %%mm1  \n\t"\
743     "movq         %%mm4, %%mm3  \n\t"\
744 \
745     "punpcklbw    %%mm7, %%mm3  \n\t"\
746     "punpcklbw    %%mm5, %%mm2  \n\t"\
747     "punpckhbw    %%mm7, %%mm4  \n\t"\
748     "punpckhbw    %%mm5, %%mm1  \n\t"\
749 \
750     "psllq           $2, %%mm3  \n\t"\
751     "psllq           $2, %%mm4  \n\t"\
752 \
753     "por          %%mm3, %%mm2  \n\t"\
754     "por          %%mm4, %%mm1  \n\t"\
755 \
756     MOVNTQ(%%mm2,  (dst, index, 2))\
757     MOVNTQ(%%mm1, 8(dst, index, 2))\
758 \
759     "add             $8, "#index"   \n\t"\
760     "cmp        "#dstw", "#index"   \n\t"\
761     " jb             1b             \n\t"
762 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
763
764 #define WRITEBGR24OLD(dst, dstw, index) \
765     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
766     "movq      %%mm2, %%mm1             \n\t" /* B */\
767     "movq      %%mm5, %%mm6             \n\t" /* R */\
768     "punpcklbw %%mm4, %%mm2             \n\t" /* GBGBGBGB 0 */\
769     "punpcklbw %%mm7, %%mm5             \n\t" /* 0R0R0R0R 0 */\
770     "punpckhbw %%mm4, %%mm1             \n\t" /* GBGBGBGB 2 */\
771     "punpckhbw %%mm7, %%mm6             \n\t" /* 0R0R0R0R 2 */\
772     "movq      %%mm2, %%mm0             \n\t" /* GBGBGBGB 0 */\
773     "movq      %%mm1, %%mm3             \n\t" /* GBGBGBGB 2 */\
774     "punpcklwd %%mm5, %%mm0             \n\t" /* 0RGB0RGB 0 */\
775     "punpckhwd %%mm5, %%mm2             \n\t" /* 0RGB0RGB 1 */\
776     "punpcklwd %%mm6, %%mm1             \n\t" /* 0RGB0RGB 2 */\
777     "punpckhwd %%mm6, %%mm3             \n\t" /* 0RGB0RGB 3 */\
778 \
779     "movq      %%mm0, %%mm4             \n\t" /* 0RGB0RGB 0 */\
780     "psrlq        $8, %%mm0             \n\t" /* 00RGB0RG 0 */\
781     "pand "MANGLE(bm00000111)", %%mm4   \n\t" /* 00000RGB 0 */\
782     "pand "MANGLE(bm11111000)", %%mm0   \n\t" /* 00RGB000 0.5 */\
783     "por       %%mm4, %%mm0             \n\t" /* 00RGBRGB 0 */\
784     "movq      %%mm2, %%mm4             \n\t" /* 0RGB0RGB 1 */\
785     "psllq       $48, %%mm2             \n\t" /* GB000000 1 */\
786     "por       %%mm2, %%mm0             \n\t" /* GBRGBRGB 0 */\
787 \
788     "movq      %%mm4, %%mm2             \n\t" /* 0RGB0RGB 1 */\
789     "psrld       $16, %%mm4             \n\t" /* 000R000R 1 */\
790     "psrlq       $24, %%mm2             \n\t" /* 0000RGB0 1.5 */\
791     "por       %%mm4, %%mm2             \n\t" /* 000RRGBR 1 */\
792     "pand "MANGLE(bm00001111)", %%mm2   \n\t" /* 0000RGBR 1 */\
793     "movq      %%mm1, %%mm4             \n\t" /* 0RGB0RGB 2 */\
794     "psrlq        $8, %%mm1             \n\t" /* 00RGB0RG 2 */\
795     "pand "MANGLE(bm00000111)", %%mm4   \n\t" /* 00000RGB 2 */\
796     "pand "MANGLE(bm11111000)", %%mm1   \n\t" /* 00RGB000 2.5 */\
797     "por       %%mm4, %%mm1             \n\t" /* 00RGBRGB 2 */\
798     "movq      %%mm1, %%mm4             \n\t" /* 00RGBRGB 2 */\
799     "psllq       $32, %%mm1             \n\t" /* BRGB0000 2 */\
800     "por       %%mm1, %%mm2             \n\t" /* BRGBRGBR 1 */\
801 \
802     "psrlq       $32, %%mm4             \n\t" /* 000000RG 2.5 */\
803     "movq      %%mm3, %%mm5             \n\t" /* 0RGB0RGB 3 */\
804     "psrlq        $8, %%mm3             \n\t" /* 00RGB0RG 3 */\
805     "pand "MANGLE(bm00000111)", %%mm5   \n\t" /* 00000RGB 3 */\
806     "pand "MANGLE(bm11111000)", %%mm3   \n\t" /* 00RGB000 3.5 */\
807     "por       %%mm5, %%mm3             \n\t" /* 00RGBRGB 3 */\
808     "psllq       $16, %%mm3             \n\t" /* RGBRGB00 3 */\
809     "por       %%mm4, %%mm3             \n\t" /* RGBRGBRG 2.5 */\
810 \
811     MOVNTQ(%%mm0,   (dst))\
812     MOVNTQ(%%mm2,  8(dst))\
813     MOVNTQ(%%mm3, 16(dst))\
814     "add         $24, "#dst"            \n\t"\
815 \
816     "add          $8, "#index"          \n\t"\
817     "cmp     "#dstw", "#index"          \n\t"\
818     " jb          1b                    \n\t"
819
820 #define WRITEBGR24MMX(dst, dstw, index) \
821     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
822     "movq      %%mm2, %%mm1     \n\t" /* B */\
823     "movq      %%mm5, %%mm6     \n\t" /* R */\
824     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
825     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
826     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
827     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
828     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
829     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
830     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
831     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
832     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
833     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
834 \
835     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
836     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
837     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
838     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
839 \
840     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
841     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
842     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
843     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
844 \
845     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
846     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
847     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
848     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
849 \
850     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
851     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
852     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
853     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
854     MOVNTQ(%%mm0, (dst))\
855 \
856     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
857     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
858     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
859     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
860     MOVNTQ(%%mm6, 8(dst))\
861 \
862     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
863     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
864     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
865     MOVNTQ(%%mm5, 16(dst))\
866 \
867     "add         $24, "#dst"    \n\t"\
868 \
869     "add          $8, "#index"  \n\t"\
870     "cmp     "#dstw", "#index"  \n\t"\
871     " jb          1b            \n\t"
872
873 #define WRITEBGR24MMX2(dst, dstw, index) \
874     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
875     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
876     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
877     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
878     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
879     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
880 \
881     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
882     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
883     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
884 \
885     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
886     "por    %%mm1, %%mm6        \n\t"\
887     "por    %%mm3, %%mm6        \n\t"\
888     MOVNTQ(%%mm6, (dst))\
889 \
890     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
891     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
892     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
893     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
894 \
895     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
896     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
897     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
898 \
899     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
900     "por    %%mm3, %%mm6        \n\t"\
901     MOVNTQ(%%mm6, 8(dst))\
902 \
903     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
904     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
905     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
906 \
907     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
908     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
909     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
910 \
911     "por    %%mm1, %%mm3        \n\t"\
912     "por    %%mm3, %%mm6        \n\t"\
913     MOVNTQ(%%mm6, 16(dst))\
914 \
915     "add      $24, "#dst"       \n\t"\
916 \
917     "add       $8, "#index"     \n\t"\
918     "cmp  "#dstw", "#index"     \n\t"\
919     " jb       1b               \n\t"
920
921 #if COMPILE_TEMPLATE_MMX2
922 #undef WRITEBGR24
923 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
924 #else
925 #undef WRITEBGR24
926 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
927 #endif
928
929 #define REAL_WRITEYUY2(dst, dstw, index) \
930     "packuswb  %%mm3, %%mm3     \n\t"\
931     "packuswb  %%mm4, %%mm4     \n\t"\
932     "packuswb  %%mm7, %%mm1     \n\t"\
933     "punpcklbw %%mm4, %%mm3     \n\t"\
934     "movq      %%mm1, %%mm7     \n\t"\
935     "punpcklbw %%mm3, %%mm1     \n\t"\
936     "punpckhbw %%mm3, %%mm7     \n\t"\
937 \
938     MOVNTQ(%%mm1, (dst, index, 2))\
939     MOVNTQ(%%mm7, 8(dst, index, 2))\
940 \
941     "add          $8, "#index"  \n\t"\
942     "cmp     "#dstw", "#index"  \n\t"\
943     " jb          1b            \n\t"
944 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
945
946
947 static inline void RENAME(yuv2yuvX)(SwsContext *c, const int16_t *lumFilter, const int16_t **lumSrc, int lumFilterSize,
948                                     const int16_t *chrFilter, const int16_t **chrSrc, int chrFilterSize, const int16_t **alpSrc,
949                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest, uint8_t *aDest, long dstW, long chrDstW)
950 {
951     if(!(c->flags & SWS_BITEXACT)) {
952         if (c->flags & SWS_ACCURATE_RND) {
953             if (uDest) {
954                 YSCALEYUV2YV12X_ACCURATE(   "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
955                 YSCALEYUV2YV12X_ACCURATE(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
956             }
957             if (CONFIG_SWSCALE_ALPHA && aDest) {
958                 YSCALEYUV2YV12X_ACCURATE(   "0", ALP_MMX_FILTER_OFFSET, aDest, dstW)
959             }
960
961             YSCALEYUV2YV12X_ACCURATE("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
962         } else {
963             if (uDest) {
964                 YSCALEYUV2YV12X(   "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
965                 YSCALEYUV2YV12X(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
966             }
967             if (CONFIG_SWSCALE_ALPHA && aDest) {
968                 YSCALEYUV2YV12X(   "0", ALP_MMX_FILTER_OFFSET, aDest, dstW)
969             }
970
971             YSCALEYUV2YV12X("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
972         }
973         return;
974     }
975     yuv2yuvXinC(lumFilter, lumSrc, lumFilterSize,
976                 chrFilter, chrSrc, chrFilterSize,
977                 alpSrc, dest, uDest, vDest, aDest, dstW, chrDstW);
978 }
979
980 static inline void RENAME(yuv2nv12X)(SwsContext *c, const int16_t *lumFilter, const int16_t **lumSrc, int lumFilterSize,
981                                      const int16_t *chrFilter, const int16_t **chrSrc, int chrFilterSize,
982                                      uint8_t *dest, uint8_t *uDest, int dstW, int chrDstW, enum PixelFormat dstFormat)
983 {
984     yuv2nv12XinC(lumFilter, lumSrc, lumFilterSize,
985                  chrFilter, chrSrc, chrFilterSize,
986                  dest, uDest, dstW, chrDstW, dstFormat);
987 }
988
989 static inline void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc, const int16_t *chrSrc, const int16_t *alpSrc,
990                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest, uint8_t *aDest, long dstW, long chrDstW)
991 {
992     int i;
993     if(!(c->flags & SWS_BITEXACT)) {
994         long p= 4;
995         const uint8_t *src[4]= {alpSrc + dstW, lumSrc + dstW, chrSrc + chrDstW, chrSrc + VOFW + chrDstW};
996         uint8_t *dst[4]= {aDest, dest, uDest, vDest};
997         x86_reg counter[4]= {dstW, dstW, chrDstW, chrDstW};
998
999         if (c->flags & SWS_ACCURATE_RND) {
1000             while(p--) {
1001                 if (dst[p]) {
1002                     __asm__ volatile(
1003                         YSCALEYUV2YV121_ACCURATE
1004                         :: "r" (src[p]), "r" (dst[p] + counter[p]),
1005                         "g" (-counter[p])
1006                         : "%"REG_a
1007                     );
1008                 }
1009             }
1010         } else {
1011             while(p--) {
1012                 if (dst[p]) {
1013                     __asm__ volatile(
1014                         YSCALEYUV2YV121
1015                         :: "r" (src[p]), "r" (dst[p] + counter[p]),
1016                         "g" (-counter[p])
1017                         : "%"REG_a
1018                     );
1019                 }
1020             }
1021         }
1022         return;
1023     }
1024     for (i=0; i<dstW; i++) {
1025         int val= (lumSrc[i]+64)>>7;
1026
1027         if (val&256) {
1028             if (val<0) val=0;
1029             else       val=255;
1030         }
1031
1032         dest[i]= val;
1033     }
1034
1035     if (uDest)
1036         for (i=0; i<chrDstW; i++) {
1037             int u=(chrSrc[i       ]+64)>>7;
1038             int v=(chrSrc[i + VOFW]+64)>>7;
1039
1040             if ((u|v)&256) {
1041                 if (u<0)        u=0;
1042                 else if (u>255) u=255;
1043                 if (v<0)        v=0;
1044                 else if (v>255) v=255;
1045             }
1046
1047             uDest[i]= u;
1048             vDest[i]= v;
1049         }
1050
1051     if (CONFIG_SWSCALE_ALPHA && aDest)
1052         for (i=0; i<dstW; i++) {
1053             int val= (alpSrc[i]+64)>>7;
1054             aDest[i]= av_clip_uint8(val);
1055         }
1056 }
1057
1058
1059 /**
1060  * vertical scale YV12 to RGB
1061  */
1062 static inline void RENAME(yuv2packedX)(SwsContext *c, const int16_t *lumFilter, const int16_t **lumSrc, int lumFilterSize,
1063                                        const int16_t *chrFilter, const int16_t **chrSrc, int chrFilterSize,
1064                                        const int16_t **alpSrc, uint8_t *dest, long dstW, long dstY)
1065 {
1066     x86_reg dummy=0;
1067     x86_reg dstW_reg = dstW;
1068     if(!(c->flags & SWS_BITEXACT)) {
1069         if (c->flags & SWS_ACCURATE_RND) {
1070             switch(c->dstFormat) {
1071             case PIX_FMT_RGB32:
1072                 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1073                     YSCALEYUV2PACKEDX_ACCURATE
1074                     YSCALEYUV2RGBX
1075                     "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
1076                     "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
1077                     "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
1078                     YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
1079                     "movq               "Y_TEMP"(%0), %%mm5         \n\t"
1080                     "psraw                        $3, %%mm1         \n\t"
1081                     "psraw                        $3, %%mm7         \n\t"
1082                     "packuswb                  %%mm7, %%mm1         \n\t"
1083                     WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
1084
1085                     YSCALEYUV2PACKEDX_END
1086                 } else {
1087                     YSCALEYUV2PACKEDX_ACCURATE
1088                     YSCALEYUV2RGBX
1089                     "pcmpeqd %%mm7, %%mm7 \n\t"
1090                     WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1091
1092                     YSCALEYUV2PACKEDX_END
1093                 }
1094                 return;
1095             case PIX_FMT_BGR24:
1096                 YSCALEYUV2PACKEDX_ACCURATE
1097                 YSCALEYUV2RGBX
1098                 "pxor %%mm7, %%mm7 \n\t"
1099                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
1100                 "add %4, %%"REG_c"                        \n\t"
1101                 WRITEBGR24(%%REGc, %5, %%REGa)
1102
1103
1104                 :: "r" (&c->redDither),
1105                 "m" (dummy), "m" (dummy), "m" (dummy),
1106                 "r" (dest), "m" (dstW_reg)
1107                 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
1108                 );
1109                 return;
1110             case PIX_FMT_RGB555:
1111                 YSCALEYUV2PACKEDX_ACCURATE
1112                 YSCALEYUV2RGBX
1113                 "pxor %%mm7, %%mm7 \n\t"
1114                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1115 #ifdef DITHER1XBPP
1116                 "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
1117                 "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
1118                 "paddusb "RED_DITHER"(%0), %%mm5\n\t"
1119 #endif
1120
1121                 WRITERGB15(%4, %5, %%REGa)
1122                 YSCALEYUV2PACKEDX_END
1123                 return;
1124             case PIX_FMT_RGB565:
1125                 YSCALEYUV2PACKEDX_ACCURATE
1126                 YSCALEYUV2RGBX
1127                 "pxor %%mm7, %%mm7 \n\t"
1128                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1129 #ifdef DITHER1XBPP
1130                 "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
1131                 "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
1132                 "paddusb "RED_DITHER"(%0), %%mm5\n\t"
1133 #endif
1134
1135                 WRITERGB16(%4, %5, %%REGa)
1136                 YSCALEYUV2PACKEDX_END
1137                 return;
1138             case PIX_FMT_YUYV422:
1139                 YSCALEYUV2PACKEDX_ACCURATE
1140                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1141
1142                 "psraw $3, %%mm3    \n\t"
1143                 "psraw $3, %%mm4    \n\t"
1144                 "psraw $3, %%mm1    \n\t"
1145                 "psraw $3, %%mm7    \n\t"
1146                 WRITEYUY2(%4, %5, %%REGa)
1147                 YSCALEYUV2PACKEDX_END
1148                 return;
1149             }
1150         } else {
1151             switch(c->dstFormat) {
1152             case PIX_FMT_RGB32:
1153                 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1154                     YSCALEYUV2PACKEDX
1155                     YSCALEYUV2RGBX
1156                     YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
1157                     "psraw                        $3, %%mm1         \n\t"
1158                     "psraw                        $3, %%mm7         \n\t"
1159                     "packuswb                  %%mm7, %%mm1         \n\t"
1160                     WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1161                     YSCALEYUV2PACKEDX_END
1162                 } else {
1163                     YSCALEYUV2PACKEDX
1164                     YSCALEYUV2RGBX
1165                     "pcmpeqd %%mm7, %%mm7 \n\t"
1166                     WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1167                     YSCALEYUV2PACKEDX_END
1168                 }
1169                 return;
1170             case PIX_FMT_BGR24:
1171                 YSCALEYUV2PACKEDX
1172                 YSCALEYUV2RGBX
1173                 "pxor                    %%mm7, %%mm7       \n\t"
1174                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
1175                 "add                        %4, %%"REG_c"   \n\t"
1176                 WRITEBGR24(%%REGc, %5, %%REGa)
1177
1178                 :: "r" (&c->redDither),
1179                 "m" (dummy), "m" (dummy), "m" (dummy),
1180                 "r" (dest),  "m" (dstW_reg)
1181                 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
1182                 );
1183                 return;
1184             case PIX_FMT_RGB555:
1185                 YSCALEYUV2PACKEDX
1186                 YSCALEYUV2RGBX
1187                 "pxor %%mm7, %%mm7 \n\t"
1188                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1189 #ifdef DITHER1XBPP
1190                 "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
1191                 "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
1192                 "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
1193 #endif
1194
1195                 WRITERGB15(%4, %5, %%REGa)
1196                 YSCALEYUV2PACKEDX_END
1197                 return;
1198             case PIX_FMT_RGB565:
1199                 YSCALEYUV2PACKEDX
1200                 YSCALEYUV2RGBX
1201                 "pxor %%mm7, %%mm7 \n\t"
1202                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1203 #ifdef DITHER1XBPP
1204                 "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
1205                 "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
1206                 "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
1207 #endif
1208
1209                 WRITERGB16(%4, %5, %%REGa)
1210                 YSCALEYUV2PACKEDX_END
1211                 return;
1212             case PIX_FMT_YUYV422:
1213                 YSCALEYUV2PACKEDX
1214                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1215
1216                 "psraw $3, %%mm3    \n\t"
1217                 "psraw $3, %%mm4    \n\t"
1218                 "psraw $3, %%mm1    \n\t"
1219                 "psraw $3, %%mm7    \n\t"
1220                 WRITEYUY2(%4, %5, %%REGa)
1221                 YSCALEYUV2PACKEDX_END
1222                 return;
1223             }
1224         }
1225     }
1226     yuv2packedXinC(c, lumFilter, lumSrc, lumFilterSize,
1227                    chrFilter, chrSrc, chrFilterSize,
1228                    alpSrc, dest, dstW, dstY);
1229 }
1230
1231 /**
1232  * vertical bilinear scale YV12 to RGB
1233  */
1234 static inline void RENAME(yuv2packed2)(SwsContext *c, const uint16_t *buf0, const uint16_t *buf1, const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1235                           const uint16_t *abuf0, const uint16_t *abuf1, uint8_t *dest, int dstW, int yalpha, int uvalpha, int y)
1236 {
1237     int  yalpha1=4095- yalpha;
1238     int uvalpha1=4095-uvalpha;
1239     int i;
1240
1241     if(!(c->flags & SWS_BITEXACT)) {
1242         switch(c->dstFormat) {
1243         //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1244         case PIX_FMT_RGB32:
1245             if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1246 #if ARCH_X86_64
1247                 __asm__ volatile(
1248                     YSCALEYUV2RGB(%%r8, %5)
1249                     YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
1250                     "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1251                     "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1252                     "packuswb            %%mm7, %%mm1       \n\t"
1253                     WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1254
1255                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "r" (dest),
1256                     "a" (&c->redDither)
1257                     ,"r" (abuf0), "r" (abuf1)
1258                     : "%r8"
1259                 );
1260 #else
1261                 *(const uint16_t **)(&c->u_temp)=abuf0;
1262                 *(const uint16_t **)(&c->v_temp)=abuf1;
1263                 __asm__ volatile(
1264                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1265                     "mov        %4, %%"REG_b"               \n\t"
1266                     "push %%"REG_BP"                        \n\t"
1267                     YSCALEYUV2RGB(%%REGBP, %5)
1268                     "push                   %0              \n\t"
1269                     "push                   %1              \n\t"
1270                     "mov          "U_TEMP"(%5), %0          \n\t"
1271                     "mov          "V_TEMP"(%5), %1          \n\t"
1272                     YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
1273                     "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1274                     "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1275                     "packuswb            %%mm7, %%mm1       \n\t"
1276                     "pop                    %1              \n\t"
1277                     "pop                    %0              \n\t"
1278                     WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1279                     "pop %%"REG_BP"                         \n\t"
1280                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1281
1282                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1283                     "a" (&c->redDither)
1284                 );
1285 #endif
1286             } else {
1287                 __asm__ volatile(
1288                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1289                     "mov        %4, %%"REG_b"               \n\t"
1290                     "push %%"REG_BP"                        \n\t"
1291                     YSCALEYUV2RGB(%%REGBP, %5)
1292                     "pcmpeqd %%mm7, %%mm7                   \n\t"
1293                     WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1294                     "pop %%"REG_BP"                         \n\t"
1295                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1296
1297                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1298                     "a" (&c->redDither)
1299                 );
1300             }
1301             return;
1302         case PIX_FMT_BGR24:
1303             __asm__ volatile(
1304                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1305                 "mov        %4, %%"REG_b"               \n\t"
1306                 "push %%"REG_BP"                        \n\t"
1307                 YSCALEYUV2RGB(%%REGBP, %5)
1308                 "pxor    %%mm7, %%mm7                   \n\t"
1309                 WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1310                 "pop %%"REG_BP"                         \n\t"
1311                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1312                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1313                 "a" (&c->redDither)
1314             );
1315             return;
1316         case PIX_FMT_RGB555:
1317             __asm__ volatile(
1318                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1319                 "mov        %4, %%"REG_b"               \n\t"
1320                 "push %%"REG_BP"                        \n\t"
1321                 YSCALEYUV2RGB(%%REGBP, %5)
1322                 "pxor    %%mm7, %%mm7                   \n\t"
1323                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1324 #ifdef DITHER1XBPP
1325                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1326                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1327                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1328 #endif
1329
1330                 WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1331                 "pop %%"REG_BP"                         \n\t"
1332                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1333
1334                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1335                 "a" (&c->redDither)
1336             );
1337             return;
1338         case PIX_FMT_RGB565:
1339             __asm__ volatile(
1340                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1341                 "mov        %4, %%"REG_b"               \n\t"
1342                 "push %%"REG_BP"                        \n\t"
1343                 YSCALEYUV2RGB(%%REGBP, %5)
1344                 "pxor    %%mm7, %%mm7                   \n\t"
1345                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1346 #ifdef DITHER1XBPP
1347                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1348                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1349                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1350 #endif
1351
1352                 WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1353                 "pop %%"REG_BP"                         \n\t"
1354                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1355                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1356                 "a" (&c->redDither)
1357             );
1358             return;
1359         case PIX_FMT_YUYV422:
1360             __asm__ volatile(
1361                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1362                 "mov %4, %%"REG_b"                        \n\t"
1363                 "push %%"REG_BP"                        \n\t"
1364                 YSCALEYUV2PACKED(%%REGBP, %5)
1365                 WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1366                 "pop %%"REG_BP"                         \n\t"
1367                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1368                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1369                 "a" (&c->redDither)
1370             );
1371             return;
1372         default: break;
1373         }
1374     }
1375     YSCALE_YUV_2_ANYRGB_C(YSCALE_YUV_2_RGB2_C, YSCALE_YUV_2_PACKED2_C(void,0), YSCALE_YUV_2_GRAY16_2_C, YSCALE_YUV_2_MONO2_C)
1376 }
1377
1378 /**
1379  * YV12 to RGB without scaling or interpolating
1380  */
1381 static inline void RENAME(yuv2packed1)(SwsContext *c, const uint16_t *buf0, const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1382                           const uint16_t *abuf0, uint8_t *dest, int dstW, int uvalpha, enum PixelFormat dstFormat, int flags, int y)
1383 {
1384     const int yalpha1=0;
1385     int i;
1386
1387     const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1388     const int yalpha= 4096; //FIXME ...
1389
1390     if (flags&SWS_FULL_CHR_H_INT) {
1391         c->yuv2packed2(c, buf0, buf0, uvbuf0, uvbuf1, abuf0, abuf0, dest, dstW, 0, uvalpha, y);
1392         return;
1393     }
1394
1395     if(!(flags & SWS_BITEXACT)) {
1396         if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1397             switch(dstFormat) {
1398             case PIX_FMT_RGB32:
1399                 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1400                     __asm__ volatile(
1401                         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1402                         "mov        %4, %%"REG_b"               \n\t"
1403                         "push %%"REG_BP"                        \n\t"
1404                         YSCALEYUV2RGB1(%%REGBP, %5)
1405                         YSCALEYUV2RGB1_ALPHA(%%REGBP)
1406                         WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1407                         "pop %%"REG_BP"                         \n\t"
1408                         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1409
1410                         :: "c" (buf0), "d" (abuf0), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1411                         "a" (&c->redDither)
1412                     );
1413                 } else {
1414                     __asm__ volatile(
1415                         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1416                         "mov        %4, %%"REG_b"               \n\t"
1417                         "push %%"REG_BP"                        \n\t"
1418                         YSCALEYUV2RGB1(%%REGBP, %5)
1419                         "pcmpeqd %%mm7, %%mm7                   \n\t"
1420                         WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1421                         "pop %%"REG_BP"                         \n\t"
1422                         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1423
1424                         :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1425                         "a" (&c->redDither)
1426                     );
1427                 }
1428                 return;
1429             case PIX_FMT_BGR24:
1430                 __asm__ volatile(
1431                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1432                     "mov        %4, %%"REG_b"               \n\t"
1433                     "push %%"REG_BP"                        \n\t"
1434                     YSCALEYUV2RGB1(%%REGBP, %5)
1435                     "pxor    %%mm7, %%mm7                   \n\t"
1436                     WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1437                     "pop %%"REG_BP"                         \n\t"
1438                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1439
1440                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1441                     "a" (&c->redDither)
1442                 );
1443                 return;
1444             case PIX_FMT_RGB555:
1445                 __asm__ volatile(
1446                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1447                     "mov        %4, %%"REG_b"               \n\t"
1448                     "push %%"REG_BP"                        \n\t"
1449                     YSCALEYUV2RGB1(%%REGBP, %5)
1450                     "pxor    %%mm7, %%mm7                   \n\t"
1451                     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1452 #ifdef DITHER1XBPP
1453                     "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1454                     "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1455                     "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1456 #endif
1457                     WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1458                     "pop %%"REG_BP"                         \n\t"
1459                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1460
1461                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1462                     "a" (&c->redDither)
1463                 );
1464                 return;
1465             case PIX_FMT_RGB565:
1466                 __asm__ volatile(
1467                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1468                     "mov        %4, %%"REG_b"               \n\t"
1469                     "push %%"REG_BP"                        \n\t"
1470                     YSCALEYUV2RGB1(%%REGBP, %5)
1471                     "pxor    %%mm7, %%mm7                   \n\t"
1472                     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1473 #ifdef DITHER1XBPP
1474                     "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1475                     "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1476                     "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1477 #endif
1478
1479                     WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1480                     "pop %%"REG_BP"                         \n\t"
1481                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1482
1483                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1484                     "a" (&c->redDither)
1485                 );
1486                 return;
1487             case PIX_FMT_YUYV422:
1488                 __asm__ volatile(
1489                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1490                     "mov        %4, %%"REG_b"               \n\t"
1491                     "push %%"REG_BP"                        \n\t"
1492                     YSCALEYUV2PACKED1(%%REGBP, %5)
1493                     WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1494                     "pop %%"REG_BP"                         \n\t"
1495                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1496
1497                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1498                     "a" (&c->redDither)
1499                 );
1500                 return;
1501             }
1502         } else {
1503             switch(dstFormat) {
1504             case PIX_FMT_RGB32:
1505                 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1506                     __asm__ volatile(
1507                         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1508                         "mov        %4, %%"REG_b"               \n\t"
1509                         "push %%"REG_BP"                        \n\t"
1510                         YSCALEYUV2RGB1b(%%REGBP, %5)
1511                         YSCALEYUV2RGB1_ALPHA(%%REGBP)
1512                         WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1513                         "pop %%"REG_BP"                         \n\t"
1514                         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1515
1516                         :: "c" (buf0), "d" (abuf0), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1517                         "a" (&c->redDither)
1518                     );
1519                 } else {
1520                     __asm__ volatile(
1521                         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1522                         "mov        %4, %%"REG_b"               \n\t"
1523                         "push %%"REG_BP"                        \n\t"
1524                         YSCALEYUV2RGB1b(%%REGBP, %5)
1525                         "pcmpeqd %%mm7, %%mm7                   \n\t"
1526                         WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1527                         "pop %%"REG_BP"                         \n\t"
1528                         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1529
1530                         :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1531                         "a" (&c->redDither)
1532                     );
1533                 }
1534                 return;
1535             case PIX_FMT_BGR24:
1536                 __asm__ volatile(
1537                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1538                     "mov        %4, %%"REG_b"               \n\t"
1539                     "push %%"REG_BP"                        \n\t"
1540                     YSCALEYUV2RGB1b(%%REGBP, %5)
1541                     "pxor    %%mm7, %%mm7                   \n\t"
1542                     WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1543                     "pop %%"REG_BP"                         \n\t"
1544                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1545
1546                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1547                     "a" (&c->redDither)
1548                 );
1549                 return;
1550             case PIX_FMT_RGB555:
1551                 __asm__ volatile(
1552                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1553                     "mov        %4, %%"REG_b"               \n\t"
1554                     "push %%"REG_BP"                        \n\t"
1555                     YSCALEYUV2RGB1b(%%REGBP, %5)
1556                     "pxor    %%mm7, %%mm7                   \n\t"
1557                     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1558 #ifdef DITHER1XBPP
1559                     "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1560                     "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1561                     "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1562 #endif
1563                     WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1564                     "pop %%"REG_BP"                         \n\t"
1565                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1566
1567                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1568                     "a" (&c->redDither)
1569                 );
1570                 return;
1571             case PIX_FMT_RGB565:
1572                 __asm__ volatile(
1573                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1574                     "mov        %4, %%"REG_b"               \n\t"
1575                     "push %%"REG_BP"                        \n\t"
1576                     YSCALEYUV2RGB1b(%%REGBP, %5)
1577                     "pxor    %%mm7, %%mm7                   \n\t"
1578                     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1579 #ifdef DITHER1XBPP
1580                     "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1581                     "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1582                     "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1583 #endif
1584
1585                     WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1586                     "pop %%"REG_BP"                         \n\t"
1587                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1588
1589                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1590                     "a" (&c->redDither)
1591                 );
1592                 return;
1593             case PIX_FMT_YUYV422:
1594                 __asm__ volatile(
1595                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1596                     "mov        %4, %%"REG_b"               \n\t"
1597                     "push %%"REG_BP"                        \n\t"
1598                     YSCALEYUV2PACKED1b(%%REGBP, %5)
1599                     WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1600                     "pop %%"REG_BP"                         \n\t"
1601                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1602
1603                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1604                     "a" (&c->redDither)
1605                 );
1606                 return;
1607             }
1608         }
1609     }
1610     if (uvalpha < 2048) {
1611         YSCALE_YUV_2_ANYRGB_C(YSCALE_YUV_2_RGB1_C, YSCALE_YUV_2_PACKED1_C(void,0), YSCALE_YUV_2_GRAY16_1_C, YSCALE_YUV_2_MONO2_C)
1612     } else {
1613         YSCALE_YUV_2_ANYRGB_C(YSCALE_YUV_2_RGB1B_C, YSCALE_YUV_2_PACKED1B_C(void,0), YSCALE_YUV_2_GRAY16_1_C, YSCALE_YUV_2_MONO2_C)
1614     }
1615 }
1616
1617 //FIXME yuy2* can read up to 7 samples too much
1618
1619 static inline void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1620 {
1621     __asm__ volatile(
1622         "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1623         "mov                    %0, %%"REG_a"       \n\t"
1624         "1:                                         \n\t"
1625         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1626         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1627         "pand                %%mm2, %%mm0           \n\t"
1628         "pand                %%mm2, %%mm1           \n\t"
1629         "packuswb            %%mm1, %%mm0           \n\t"
1630         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1631         "add                    $8, %%"REG_a"       \n\t"
1632         " js                    1b                  \n\t"
1633         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1634         : "%"REG_a
1635     );
1636 }
1637
1638 static inline void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1639 {
1640     __asm__ volatile(
1641         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1642         "mov                    %0, %%"REG_a"       \n\t"
1643         "1:                                         \n\t"
1644         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1645         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1646         "psrlw                  $8, %%mm0           \n\t"
1647         "psrlw                  $8, %%mm1           \n\t"
1648         "packuswb            %%mm1, %%mm0           \n\t"
1649         "movq                %%mm0, %%mm1           \n\t"
1650         "psrlw                  $8, %%mm0           \n\t"
1651         "pand                %%mm4, %%mm1           \n\t"
1652         "packuswb            %%mm0, %%mm0           \n\t"
1653         "packuswb            %%mm1, %%mm1           \n\t"
1654         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1655         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1656         "add                    $4, %%"REG_a"       \n\t"
1657         " js                    1b                  \n\t"
1658         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1659         : "%"REG_a
1660     );
1661     assert(src1 == src2);
1662 }
1663
1664 static inline void RENAME(LEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1665 {
1666     __asm__ volatile(
1667         "mov                    %0, %%"REG_a"       \n\t"
1668         "1:                                         \n\t"
1669         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1670         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1671         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1672         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1673         "psrlw                  $8, %%mm0           \n\t"
1674         "psrlw                  $8, %%mm1           \n\t"
1675         "psrlw                  $8, %%mm2           \n\t"
1676         "psrlw                  $8, %%mm3           \n\t"
1677         "packuswb            %%mm1, %%mm0           \n\t"
1678         "packuswb            %%mm3, %%mm2           \n\t"
1679         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1680         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1681         "add                    $8, %%"REG_a"       \n\t"
1682         " js                    1b                  \n\t"
1683         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1684         : "%"REG_a
1685     );
1686 }
1687
1688 /* This is almost identical to the previous, end exists only because
1689  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1690 static inline void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1691 {
1692     __asm__ volatile(
1693         "mov                  %0, %%"REG_a"         \n\t"
1694         "1:                                         \n\t"
1695         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1696         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1697         "psrlw                $8, %%mm0             \n\t"
1698         "psrlw                $8, %%mm1             \n\t"
1699         "packuswb          %%mm1, %%mm0             \n\t"
1700         "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1701         "add                  $8, %%"REG_a"         \n\t"
1702         " js                  1b                    \n\t"
1703         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1704         : "%"REG_a
1705     );
1706 }
1707
1708 static inline void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1709 {
1710     __asm__ volatile(
1711         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1712         "mov                    %0, %%"REG_a"       \n\t"
1713         "1:                                         \n\t"
1714         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1715         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1716         "pand                %%mm4, %%mm0           \n\t"
1717         "pand                %%mm4, %%mm1           \n\t"
1718         "packuswb            %%mm1, %%mm0           \n\t"
1719         "movq                %%mm0, %%mm1           \n\t"
1720         "psrlw                  $8, %%mm0           \n\t"
1721         "pand                %%mm4, %%mm1           \n\t"
1722         "packuswb            %%mm0, %%mm0           \n\t"
1723         "packuswb            %%mm1, %%mm1           \n\t"
1724         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1725         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1726         "add                    $4, %%"REG_a"       \n\t"
1727         " js                    1b                  \n\t"
1728         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1729         : "%"REG_a
1730     );
1731     assert(src1 == src2);
1732 }
1733
1734 static inline void RENAME(BEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1735 {
1736     __asm__ volatile(
1737         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1738         "mov                    %0, %%"REG_a"       \n\t"
1739         "1:                                         \n\t"
1740         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1741         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1742         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1743         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1744         "pand                %%mm4, %%mm0           \n\t"
1745         "pand                %%mm4, %%mm1           \n\t"
1746         "pand                %%mm4, %%mm2           \n\t"
1747         "pand                %%mm4, %%mm3           \n\t"
1748         "packuswb            %%mm1, %%mm0           \n\t"
1749         "packuswb            %%mm3, %%mm2           \n\t"
1750         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1751         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1752         "add                    $8, %%"REG_a"       \n\t"
1753         " js                    1b                  \n\t"
1754         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1755         : "%"REG_a
1756     );
1757 }
1758
1759 static inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1760                                     const uint8_t *src, long width)
1761 {
1762     __asm__ volatile(
1763         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1764         "mov                    %0, %%"REG_a"       \n\t"
1765         "1:                                         \n\t"
1766         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1767         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1768         "movq                %%mm0, %%mm2           \n\t"
1769         "movq                %%mm1, %%mm3           \n\t"
1770         "pand                %%mm4, %%mm0           \n\t"
1771         "pand                %%mm4, %%mm1           \n\t"
1772         "psrlw                  $8, %%mm2           \n\t"
1773         "psrlw                  $8, %%mm3           \n\t"
1774         "packuswb            %%mm1, %%mm0           \n\t"
1775         "packuswb            %%mm3, %%mm2           \n\t"
1776         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1777         "movq                %%mm2, (%3, %%"REG_a") \n\t"
1778         "add                    $8, %%"REG_a"       \n\t"
1779         " js                    1b                  \n\t"
1780         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1781         : "%"REG_a
1782     );
1783 }
1784
1785 static inline void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1786                                     const uint8_t *src1, const uint8_t *src2,
1787                                     long width, uint32_t *unused)
1788 {
1789     RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1790 }
1791
1792 static inline void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1793                                     const uint8_t *src1, const uint8_t *src2,
1794                                     long width, uint32_t *unused)
1795 {
1796     RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1797 }
1798
1799 static inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, const uint8_t *src, long width, enum PixelFormat srcFormat)
1800 {
1801
1802     if(srcFormat == PIX_FMT_BGR24) {
1803         __asm__ volatile(
1804             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1805             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1806             :
1807         );
1808     } else {
1809         __asm__ volatile(
1810             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1811             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1812             :
1813         );
1814     }
1815
1816     __asm__ volatile(
1817         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1818         "mov                        %2, %%"REG_a"   \n\t"
1819         "pxor                    %%mm7, %%mm7       \n\t"
1820         "1:                                         \n\t"
1821         PREFETCH"               64(%0)              \n\t"
1822         "movd                     (%0), %%mm0       \n\t"
1823         "movd                    2(%0), %%mm1       \n\t"
1824         "movd                    6(%0), %%mm2       \n\t"
1825         "movd                    8(%0), %%mm3       \n\t"
1826         "add                       $12, %0          \n\t"
1827         "punpcklbw               %%mm7, %%mm0       \n\t"
1828         "punpcklbw               %%mm7, %%mm1       \n\t"
1829         "punpcklbw               %%mm7, %%mm2       \n\t"
1830         "punpcklbw               %%mm7, %%mm3       \n\t"
1831         "pmaddwd                 %%mm5, %%mm0       \n\t"
1832         "pmaddwd                 %%mm6, %%mm1       \n\t"
1833         "pmaddwd                 %%mm5, %%mm2       \n\t"
1834         "pmaddwd                 %%mm6, %%mm3       \n\t"
1835         "paddd                   %%mm1, %%mm0       \n\t"
1836         "paddd                   %%mm3, %%mm2       \n\t"
1837         "paddd                   %%mm4, %%mm0       \n\t"
1838         "paddd                   %%mm4, %%mm2       \n\t"
1839         "psrad                     $15, %%mm0       \n\t"
1840         "psrad                     $15, %%mm2       \n\t"
1841         "packssdw                %%mm2, %%mm0       \n\t"
1842         "packuswb                %%mm0, %%mm0       \n\t"
1843         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1844         "add                        $4, %%"REG_a"   \n\t"
1845         " js                        1b              \n\t"
1846     : "+r" (src)
1847     : "r" (dst+width), "g" ((x86_reg)-width)
1848     : "%"REG_a
1849     );
1850 }
1851
1852 static inline void RENAME(bgr24ToUV_mmx)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src, long width, enum PixelFormat srcFormat)
1853 {
1854     __asm__ volatile(
1855         "movq                    24(%4), %%mm6       \n\t"
1856         "mov                        %3, %%"REG_a"   \n\t"
1857         "pxor                    %%mm7, %%mm7       \n\t"
1858         "1:                                         \n\t"
1859         PREFETCH"               64(%0)              \n\t"
1860         "movd                     (%0), %%mm0       \n\t"
1861         "movd                    2(%0), %%mm1       \n\t"
1862         "punpcklbw               %%mm7, %%mm0       \n\t"
1863         "punpcklbw               %%mm7, %%mm1       \n\t"
1864         "movq                    %%mm0, %%mm2       \n\t"
1865         "movq                    %%mm1, %%mm3       \n\t"
1866         "pmaddwd                  (%4), %%mm0       \n\t"
1867         "pmaddwd                 8(%4), %%mm1       \n\t"
1868         "pmaddwd                16(%4), %%mm2       \n\t"
1869         "pmaddwd                 %%mm6, %%mm3       \n\t"
1870         "paddd                   %%mm1, %%mm0       \n\t"
1871         "paddd                   %%mm3, %%mm2       \n\t"
1872
1873         "movd                    6(%0), %%mm1       \n\t"
1874         "movd                    8(%0), %%mm3       \n\t"
1875         "add                       $12, %0          \n\t"
1876         "punpcklbw               %%mm7, %%mm1       \n\t"
1877         "punpcklbw               %%mm7, %%mm3       \n\t"
1878         "movq                    %%mm1, %%mm4       \n\t"
1879         "movq                    %%mm3, %%mm5       \n\t"
1880         "pmaddwd                  (%4), %%mm1       \n\t"
1881         "pmaddwd                 8(%4), %%mm3       \n\t"
1882         "pmaddwd                16(%4), %%mm4       \n\t"
1883         "pmaddwd                 %%mm6, %%mm5       \n\t"
1884         "paddd                   %%mm3, %%mm1       \n\t"
1885         "paddd                   %%mm5, %%mm4       \n\t"
1886
1887         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1888         "paddd                   %%mm3, %%mm0       \n\t"
1889         "paddd                   %%mm3, %%mm2       \n\t"
1890         "paddd                   %%mm3, %%mm1       \n\t"
1891         "paddd                   %%mm3, %%mm4       \n\t"
1892         "psrad                     $15, %%mm0       \n\t"
1893         "psrad                     $15, %%mm2       \n\t"
1894         "psrad                     $15, %%mm1       \n\t"
1895         "psrad                     $15, %%mm4       \n\t"
1896         "packssdw                %%mm1, %%mm0       \n\t"
1897         "packssdw                %%mm4, %%mm2       \n\t"
1898         "packuswb                %%mm0, %%mm0       \n\t"
1899         "packuswb                %%mm2, %%mm2       \n\t"
1900         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1901         "movd                %%mm2, (%2, %%"REG_a") \n\t"
1902         "add                        $4, %%"REG_a"   \n\t"
1903         " js                        1b              \n\t"
1904     : "+r" (src)
1905     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1906     : "%"REG_a
1907     );
1908 }
1909
1910 static inline void RENAME(bgr24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1911 {
1912     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1913 }
1914
1915 static inline void RENAME(bgr24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1916 {
1917     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1918     assert(src1 == src2);
1919 }
1920
1921 static inline void RENAME(bgr24ToUV_half)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1922 {
1923     int i;
1924     for (i=0; i<width; i++) {
1925         int b= src1[6*i + 0] + src1[6*i + 3];
1926         int g= src1[6*i + 1] + src1[6*i + 4];
1927         int r= src1[6*i + 2] + src1[6*i + 5];
1928
1929         dstU[i]= (RU*r + GU*g + BU*b + (257<<RGB2YUV_SHIFT))>>(RGB2YUV_SHIFT+1);
1930         dstV[i]= (RV*r + GV*g + BV*b + (257<<RGB2YUV_SHIFT))>>(RGB2YUV_SHIFT+1);
1931     }
1932     assert(src1 == src2);
1933 }
1934
1935 static inline void RENAME(rgb24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1936 {
1937     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1938 }
1939
1940 static inline void RENAME(rgb24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1941 {
1942     assert(src1==src2);
1943     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1944 }
1945
1946 static inline void RENAME(rgb24ToUV_half)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1947 {
1948     int i;
1949     assert(src1==src2);
1950     for (i=0; i<width; i++) {
1951         int r= src1[6*i + 0] + src1[6*i + 3];
1952         int g= src1[6*i + 1] + src1[6*i + 4];
1953         int b= src1[6*i + 2] + src1[6*i + 5];
1954
1955         dstU[i]= (RU*r + GU*g + BU*b + (257<<RGB2YUV_SHIFT))>>(RGB2YUV_SHIFT+1);
1956         dstV[i]= (RV*r + GV*g + BV*b + (257<<RGB2YUV_SHIFT))>>(RGB2YUV_SHIFT+1);
1957     }
1958 }
1959
1960
1961 // bilinear / bicubic scaling
1962 static inline void RENAME(hScale)(int16_t *dst, int dstW, const uint8_t *src, int srcW, int xInc,
1963                                   const int16_t *filter, const int16_t *filterPos, long filterSize)
1964 {
1965     assert(filterSize % 4 == 0 && filterSize>0);
1966     if (filterSize==4) { // Always true for upscaling, sometimes for down, too.
1967         x86_reg counter= -2*dstW;
1968         filter-= counter*2;
1969         filterPos-= counter/2;
1970         dst-= counter/2;
1971         __asm__ volatile(
1972 #if defined(PIC)
1973             "push            %%"REG_b"              \n\t"
1974 #endif
1975             "pxor                %%mm7, %%mm7       \n\t"
1976             "push           %%"REG_BP"              \n\t" // we use 7 regs here ...
1977             "mov             %%"REG_a", %%"REG_BP"  \n\t"
1978             ".p2align                4              \n\t"
1979             "1:                                     \n\t"
1980             "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"
1981             "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"
1982             "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"
1983             "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"
1984             "movd      (%3, %%"REG_a"), %%mm0       \n\t"
1985             "movd      (%3, %%"REG_b"), %%mm2       \n\t"
1986             "punpcklbw           %%mm7, %%mm0       \n\t"
1987             "punpcklbw           %%mm7, %%mm2       \n\t"
1988             "pmaddwd             %%mm1, %%mm0       \n\t"
1989             "pmaddwd             %%mm2, %%mm3       \n\t"
1990             "movq                %%mm0, %%mm4       \n\t"
1991             "punpckldq           %%mm3, %%mm0       \n\t"
1992             "punpckhdq           %%mm3, %%mm4       \n\t"
1993             "paddd               %%mm4, %%mm0       \n\t"
1994             "psrad                  $7, %%mm0       \n\t"
1995             "packssdw            %%mm0, %%mm0       \n\t"
1996             "movd                %%mm0, (%4, %%"REG_BP")    \n\t"
1997             "add                    $4, %%"REG_BP"  \n\t"
1998             " jnc                   1b              \n\t"
1999
2000             "pop            %%"REG_BP"              \n\t"
2001 #if defined(PIC)
2002             "pop             %%"REG_b"              \n\t"
2003 #endif
2004             : "+a" (counter)
2005             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
2006 #if !defined(PIC)
2007             : "%"REG_b
2008 #endif
2009         );
2010     } else if (filterSize==8) {
2011         x86_reg counter= -2*dstW;
2012         filter-= counter*4;
2013         filterPos-= counter/2;
2014         dst-= counter/2;
2015         __asm__ volatile(
2016 #if defined(PIC)
2017             "push             %%"REG_b"             \n\t"
2018 #endif
2019             "pxor                 %%mm7, %%mm7      \n\t"
2020             "push            %%"REG_BP"             \n\t" // we use 7 regs here ...
2021             "mov              %%"REG_a", %%"REG_BP" \n\t"
2022             ".p2align                 4             \n\t"
2023             "1:                                     \n\t"
2024             "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"
2025             "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"
2026             "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"
2027             "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"
2028             "movd       (%3, %%"REG_a"), %%mm0      \n\t"
2029             "movd       (%3, %%"REG_b"), %%mm2      \n\t"
2030             "punpcklbw            %%mm7, %%mm0      \n\t"
2031             "punpcklbw            %%mm7, %%mm2      \n\t"
2032             "pmaddwd              %%mm1, %%mm0      \n\t"
2033             "pmaddwd              %%mm2, %%mm3      \n\t"
2034
2035             "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"
2036             "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"
2037             "movd      4(%3, %%"REG_a"), %%mm4      \n\t"
2038             "movd      4(%3, %%"REG_b"), %%mm2      \n\t"
2039             "punpcklbw            %%mm7, %%mm4      \n\t"
2040             "punpcklbw            %%mm7, %%mm2      \n\t"
2041             "pmaddwd              %%mm1, %%mm4      \n\t"
2042             "pmaddwd              %%mm2, %%mm5      \n\t"
2043             "paddd                %%mm4, %%mm0      \n\t"
2044             "paddd                %%mm5, %%mm3      \n\t"
2045             "movq                 %%mm0, %%mm4      \n\t"
2046             "punpckldq            %%mm3, %%mm0      \n\t"
2047             "punpckhdq            %%mm3, %%mm4      \n\t"
2048             "paddd                %%mm4, %%mm0      \n\t"
2049             "psrad                   $7, %%mm0      \n\t"
2050             "packssdw             %%mm0, %%mm0      \n\t"
2051             "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"
2052             "add                     $4, %%"REG_BP" \n\t"
2053             " jnc                    1b             \n\t"
2054
2055             "pop             %%"REG_BP"             \n\t"
2056 #if defined(PIC)
2057             "pop              %%"REG_b"             \n\t"
2058 #endif
2059             : "+a" (counter)
2060             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
2061 #if !defined(PIC)
2062             : "%"REG_b
2063 #endif
2064         );
2065     } else {
2066         const uint8_t *offset = src+filterSize;
2067         x86_reg counter= -2*dstW;
2068         //filter-= counter*filterSize/2;
2069         filterPos-= counter/2;
2070         dst-= counter/2;
2071         __asm__ volatile(
2072             "pxor                  %%mm7, %%mm7     \n\t"
2073             ".p2align                  4            \n\t"
2074             "1:                                     \n\t"
2075             "mov                      %2, %%"REG_c" \n\t"
2076             "movzwl      (%%"REG_c", %0), %%eax     \n\t"
2077             "movzwl     2(%%"REG_c", %0), %%edx     \n\t"
2078             "mov                      %5, %%"REG_c" \n\t"
2079             "pxor                  %%mm4, %%mm4     \n\t"
2080             "pxor                  %%mm5, %%mm5     \n\t"
2081             "2:                                     \n\t"
2082             "movq                   (%1), %%mm1     \n\t"
2083             "movq               (%1, %6), %%mm3     \n\t"
2084             "movd (%%"REG_c", %%"REG_a"), %%mm0     \n\t"
2085             "movd (%%"REG_c", %%"REG_d"), %%mm2     \n\t"
2086             "punpcklbw             %%mm7, %%mm0     \n\t"
2087             "punpcklbw             %%mm7, %%mm2     \n\t"
2088             "pmaddwd               %%mm1, %%mm0     \n\t"
2089             "pmaddwd               %%mm2, %%mm3     \n\t"
2090             "paddd                 %%mm3, %%mm5     \n\t"
2091             "paddd                 %%mm0, %%mm4     \n\t"
2092             "add                      $8, %1        \n\t"
2093             "add                      $4, %%"REG_c" \n\t"
2094             "cmp                      %4, %%"REG_c" \n\t"
2095             " jb                      2b            \n\t"
2096             "add                      %6, %1        \n\t"
2097             "movq                  %%mm4, %%mm0     \n\t"
2098             "punpckldq             %%mm5, %%mm4     \n\t"
2099             "punpckhdq             %%mm5, %%mm0     \n\t"
2100             "paddd                 %%mm0, %%mm4     \n\t"
2101             "psrad                    $7, %%mm4     \n\t"
2102             "packssdw              %%mm4, %%mm4     \n\t"
2103             "mov                      %3, %%"REG_a" \n\t"
2104             "movd                  %%mm4, (%%"REG_a", %0)   \n\t"
2105             "add                      $4, %0        \n\t"
2106             " jnc                     1b            \n\t"
2107
2108             : "+r" (counter), "+r" (filter)
2109             : "m" (filterPos), "m" (dst), "m"(offset),
2110             "m" (src), "r" ((x86_reg)filterSize*2)
2111             : "%"REG_a, "%"REG_c, "%"REG_d
2112         );
2113     }
2114 }
2115
2116 //FIXME all pal and rgb srcFormats could do this convertion as well
2117 //FIXME all scalers more complex than bilinear could do half of this transform
2118 static void RENAME(chrRangeToJpeg)(uint16_t *dst, int width)
2119 {
2120     int i;
2121     for (i = 0; i < width; i++) {
2122         dst[i     ] = (FFMIN(dst[i     ],30775)*4663 - 9289992)>>12; //-264
2123         dst[i+VOFW] = (FFMIN(dst[i+VOFW],30775)*4663 - 9289992)>>12; //-264
2124     }
2125 }
2126 static void RENAME(chrRangeFromJpeg)(uint16_t *dst, int width)
2127 {
2128     int i;
2129     for (i = 0; i < width; i++) {
2130         dst[i     ] = (dst[i     ]*1799 + 4081085)>>11; //1469
2131         dst[i+VOFW] = (dst[i+VOFW]*1799 + 4081085)>>11; //1469
2132     }
2133 }
2134 static void RENAME(lumRangeToJpeg)(uint16_t *dst, int width)
2135 {
2136     int i;
2137     for (i = 0; i < width; i++)
2138         dst[i] = (FFMIN(dst[i],30189)*19077 - 39057361)>>14;
2139 }
2140 static void RENAME(lumRangeFromJpeg)(uint16_t *dst, int width)
2141 {
2142     int i;
2143     for (i = 0; i < width; i++)
2144         dst[i] = (dst[i]*14071 + 33561947)>>14;
2145 }
2146
2147 #define FAST_BILINEAR_X86 \
2148     "subl    %%edi, %%esi    \n\t" /*  src[xx+1] - src[xx] */                   \
2149     "imull   %%ecx, %%esi    \n\t" /* (src[xx+1] - src[xx])*xalpha */           \
2150     "shll      $16, %%edi    \n\t"                                              \
2151     "addl    %%edi, %%esi    \n\t" /* src[xx+1]*xalpha + src[xx]*(1-xalpha) */  \
2152     "mov        %1, %%"REG_D"\n\t"                                              \
2153     "shrl       $9, %%esi    \n\t"                                              \
2154
2155 static inline void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
2156                                         long dstWidth, const uint8_t *src, int srcW,
2157                                         int xInc)
2158 {
2159 #if COMPILE_TEMPLATE_MMX2
2160     int32_t *filterPos = c->hLumFilterPos;
2161     int16_t *filter    = c->hLumFilter;
2162     int     canMMX2BeUsed  = c->canMMX2BeUsed;
2163     void    *mmx2FilterCode= c->lumMmx2FilterCode;
2164     int i;
2165 #if defined(PIC)
2166     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2167 #endif
2168     if (canMMX2BeUsed) {
2169         __asm__ volatile(
2170 #if defined(PIC)
2171             "mov               %%"REG_b", %5        \n\t"
2172 #endif
2173             "pxor                  %%mm7, %%mm7     \n\t"
2174             "mov                      %0, %%"REG_c" \n\t"
2175             "mov                      %1, %%"REG_D" \n\t"
2176             "mov                      %2, %%"REG_d" \n\t"
2177             "mov                      %3, %%"REG_b" \n\t"
2178             "xor               %%"REG_a", %%"REG_a" \n\t" // i
2179             PREFETCH"        (%%"REG_c")            \n\t"
2180             PREFETCH"      32(%%"REG_c")            \n\t"
2181             PREFETCH"      64(%%"REG_c")            \n\t"
2182
2183 #if ARCH_X86_64
2184
2185 #define CALL_MMX2_FILTER_CODE \
2186             "movl            (%%"REG_b"), %%esi     \n\t"\
2187             "call                    *%4            \n\t"\
2188             "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
2189             "add               %%"REG_S", %%"REG_c" \n\t"\
2190             "add               %%"REG_a", %%"REG_D" \n\t"\
2191             "xor               %%"REG_a", %%"REG_a" \n\t"\
2192
2193 #else
2194
2195 #define CALL_MMX2_FILTER_CODE \
2196             "movl (%%"REG_b"), %%esi        \n\t"\
2197             "call         *%4                       \n\t"\
2198             "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
2199             "add               %%"REG_a", %%"REG_D" \n\t"\
2200             "xor               %%"REG_a", %%"REG_a" \n\t"\
2201
2202 #endif /* ARCH_X86_64 */
2203
2204             CALL_MMX2_FILTER_CODE
2205             CALL_MMX2_FILTER_CODE
2206             CALL_MMX2_FILTER_CODE
2207             CALL_MMX2_FILTER_CODE
2208             CALL_MMX2_FILTER_CODE
2209             CALL_MMX2_FILTER_CODE
2210             CALL_MMX2_FILTER_CODE
2211             CALL_MMX2_FILTER_CODE
2212
2213 #if defined(PIC)
2214             "mov                      %5, %%"REG_b" \n\t"
2215 #endif
2216             :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
2217             "m" (mmx2FilterCode)
2218 #if defined(PIC)
2219             ,"m" (ebxsave)
2220 #endif
2221             : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2222 #if !defined(PIC)
2223             ,"%"REG_b
2224 #endif
2225         );
2226         for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) dst[i] = src[srcW-1]*128;
2227     } else {
2228 #endif /* COMPILE_TEMPLATE_MMX2 */
2229     x86_reg xInc_shr16 = xInc >> 16;
2230     uint16_t xInc_mask = xInc & 0xffff;
2231     x86_reg dstWidth_reg = dstWidth;
2232     //NO MMX just normal asm ...
2233     __asm__ volatile(
2234         "xor %%"REG_a", %%"REG_a"            \n\t" // i
2235         "xor %%"REG_d", %%"REG_d"            \n\t" // xx
2236         "xorl    %%ecx, %%ecx                \n\t" // xalpha
2237         ".p2align                4           \n\t"
2238         "1:                                  \n\t"
2239         "movzbl    (%0, %%"REG_d"), %%edi    \n\t" //src[xx]
2240         "movzbl   1(%0, %%"REG_d"), %%esi    \n\t" //src[xx+1]
2241         FAST_BILINEAR_X86
2242         "movw     %%si, (%%"REG_D", %%"REG_a", 2)   \n\t"
2243         "addw       %4, %%cx                 \n\t" //xalpha += xInc&0xFFFF
2244         "adc        %3, %%"REG_d"            \n\t" //xx+= xInc>>16 + carry
2245
2246         "movzbl    (%0, %%"REG_d"), %%edi    \n\t" //src[xx]
2247         "movzbl   1(%0, %%"REG_d"), %%esi    \n\t" //src[xx+1]
2248         FAST_BILINEAR_X86
2249         "movw     %%si, 2(%%"REG_D", %%"REG_a", 2)  \n\t"
2250         "addw       %4, %%cx                 \n\t" //xalpha += xInc&0xFFFF
2251         "adc        %3, %%"REG_d"            \n\t" //xx+= xInc>>16 + carry
2252
2253
2254         "add        $2, %%"REG_a"            \n\t"
2255         "cmp        %2, %%"REG_a"            \n\t"
2256         " jb        1b                       \n\t"
2257
2258
2259         :: "r" (src), "m" (dst), "m" (dstWidth_reg), "m" (xInc_shr16), "m" (xInc_mask)
2260         : "%"REG_a, "%"REG_d, "%ecx", "%"REG_D, "%esi"
2261     );
2262 #if COMPILE_TEMPLATE_MMX2
2263     } //if MMX2 can't be used
2264 #endif
2265 }
2266
2267       // *** horizontal scale Y line to temp buffer
2268 static inline void RENAME(hyscale)(SwsContext *c, uint16_t *dst, long dstWidth, const uint8_t *src, int srcW, int xInc,
2269                                    const int16_t *hLumFilter,
2270                                    const int16_t *hLumFilterPos, int hLumFilterSize,
2271                                    uint8_t *formatConvBuffer,
2272                                    uint32_t *pal, int isAlpha)
2273 {
2274     void (*toYV12)(uint8_t *, const uint8_t *, long, uint32_t *) = isAlpha ? c->alpToYV12 : c->lumToYV12;
2275     void (*convertRange)(uint16_t *, int) = isAlpha ? NULL : c->lumConvertRange;
2276
2277     src += isAlpha ? c->alpSrcOffset : c->lumSrcOffset;
2278
2279     if (toYV12) {
2280         toYV12(formatConvBuffer, src, srcW, pal);
2281         src= formatConvBuffer;
2282     }
2283
2284     if (!c->hyscale_fast) {
2285         c->hScale(dst, dstWidth, src, srcW, xInc, hLumFilter, hLumFilterPos, hLumFilterSize);
2286     } else { // fast bilinear upscale / crap downscale
2287         c->hyscale_fast(c, dst, dstWidth, src, srcW, xInc);
2288     }
2289
2290     if (convertRange)
2291         convertRange(dst, dstWidth);
2292 }
2293
2294 static inline void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst,
2295                                         long dstWidth, const uint8_t *src1,
2296                                         const uint8_t *src2, int srcW, int xInc)
2297 {
2298 #if COMPILE_TEMPLATE_MMX2
2299     int32_t *filterPos = c->hChrFilterPos;
2300     int16_t *filter    = c->hChrFilter;
2301     int     canMMX2BeUsed  = c->canMMX2BeUsed;
2302     void    *mmx2FilterCode= c->chrMmx2FilterCode;
2303     int i;
2304 #if defined(PIC)
2305     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2306 #endif
2307     if (canMMX2BeUsed) {
2308         __asm__ volatile(
2309 #if defined(PIC)
2310             "mov          %%"REG_b", %6         \n\t"
2311 #endif
2312             "pxor             %%mm7, %%mm7      \n\t"
2313             "mov                 %0, %%"REG_c"  \n\t"
2314             "mov                 %1, %%"REG_D"  \n\t"
2315             "mov                 %2, %%"REG_d"  \n\t"
2316             "mov                 %3, %%"REG_b"  \n\t"
2317             "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2318             PREFETCH"   (%%"REG_c")             \n\t"
2319             PREFETCH" 32(%%"REG_c")             \n\t"
2320             PREFETCH" 64(%%"REG_c")             \n\t"
2321
2322             CALL_MMX2_FILTER_CODE
2323             CALL_MMX2_FILTER_CODE
2324             CALL_MMX2_FILTER_CODE
2325             CALL_MMX2_FILTER_CODE
2326             "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2327             "mov                 %5, %%"REG_c"  \n\t" // src
2328             "mov                 %1, %%"REG_D"  \n\t" // buf1
2329             "add              $"AV_STRINGIFY(VOF)", %%"REG_D"  \n\t"
2330             PREFETCH"   (%%"REG_c")             \n\t"
2331             PREFETCH" 32(%%"REG_c")             \n\t"
2332             PREFETCH" 64(%%"REG_c")             \n\t"
2333
2334             CALL_MMX2_FILTER_CODE
2335             CALL_MMX2_FILTER_CODE
2336             CALL_MMX2_FILTER_CODE
2337             CALL_MMX2_FILTER_CODE
2338
2339 #if defined(PIC)
2340             "mov %6, %%"REG_b"    \n\t"
2341 #endif
2342             :: "m" (src1), "m" (dst), "m" (filter), "m" (filterPos),
2343             "m" (mmx2FilterCode), "m" (src2)
2344 #if defined(PIC)
2345             ,"m" (ebxsave)
2346 #endif
2347             : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2348 #if !defined(PIC)
2349             ,"%"REG_b
2350 #endif
2351         );
2352         for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
2353             //printf("%d %d %d\n", dstWidth, i, srcW);
2354             dst[i] = src1[srcW-1]*128;
2355             dst[i+VOFW] = src2[srcW-1]*128;
2356         }
2357     } else {
2358 #endif /* COMPILE_TEMPLATE_MMX2 */
2359         x86_reg xInc_shr16 = (x86_reg) (xInc >> 16);
2360         uint16_t xInc_mask = xInc & 0xffff;
2361         x86_reg dstWidth_reg = dstWidth;
2362         __asm__ volatile(
2363             "xor %%"REG_a", %%"REG_a"               \n\t" // i
2364             "xor %%"REG_d", %%"REG_d"               \n\t" // xx
2365             "xorl    %%ecx, %%ecx                   \n\t" // xalpha
2366             ".p2align    4                          \n\t"
2367             "1:                                     \n\t"
2368             "mov        %0, %%"REG_S"               \n\t"
2369             "movzbl  (%%"REG_S", %%"REG_d"), %%edi  \n\t" //src[xx]
2370             "movzbl 1(%%"REG_S", %%"REG_d"), %%esi  \n\t" //src[xx+1]
2371             FAST_BILINEAR_X86
2372             "movw     %%si, (%%"REG_D", %%"REG_a", 2)   \n\t"
2373
2374             "movzbl    (%5, %%"REG_d"), %%edi       \n\t" //src[xx]
2375             "movzbl   1(%5, %%"REG_d"), %%esi       \n\t" //src[xx+1]
2376             FAST_BILINEAR_X86
2377             "movw     %%si, "AV_STRINGIFY(VOF)"(%%"REG_D", %%"REG_a", 2)   \n\t"
2378
2379             "addw       %4, %%cx                    \n\t" //xalpha += xInc&0xFFFF
2380             "adc        %3, %%"REG_d"               \n\t" //xx+= xInc>>16 + carry
2381             "add        $1, %%"REG_a"               \n\t"
2382             "cmp        %2, %%"REG_a"               \n\t"
2383             " jb        1b                          \n\t"
2384
2385 /* GCC 3.3 makes MPlayer crash on IA-32 machines when using "g" operand here,
2386 which is needed to support GCC 4.0. */
2387 #if ARCH_X86_64 && AV_GCC_VERSION_AT_LEAST(3,4)
2388             :: "m" (src1), "m" (dst), "g" (dstWidth_reg), "m" (xInc_shr16), "m" (xInc_mask),
2389 #else
2390             :: "m" (src1), "m" (dst), "m" (dstWidth_reg), "m" (xInc_shr16), "m" (xInc_mask),
2391 #endif
2392             "r" (src2)
2393             : "%"REG_a, "%"REG_d, "%ecx", "%"REG_D, "%esi"
2394         );
2395 #if COMPILE_TEMPLATE_MMX2
2396     } //if MMX2 can't be used
2397 #endif
2398 }
2399
2400 inline static void RENAME(hcscale)(SwsContext *c, uint16_t *dst, long dstWidth, const uint8_t *src1, const uint8_t *src2,
2401                                    int srcW, int xInc, const int16_t *hChrFilter,
2402                                    const int16_t *hChrFilterPos, int hChrFilterSize,
2403                                    uint8_t *formatConvBuffer,
2404                                    uint32_t *pal)
2405 {
2406
2407     src1 += c->chrSrcOffset;
2408     src2 += c->chrSrcOffset;
2409
2410     if (c->chrToYV12) {
2411         c->chrToYV12(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2412         src1= formatConvBuffer;
2413         src2= formatConvBuffer+VOFW;
2414     }
2415
2416     if (!c->hcscale_fast) {
2417         c->hScale(dst     , dstWidth, src1, srcW, xInc, hChrFilter, hChrFilterPos, hChrFilterSize);
2418         c->hScale(dst+VOFW, dstWidth, src2, srcW, xInc, hChrFilter, hChrFilterPos, hChrFilterSize);
2419     } else { // fast bilinear upscale / crap downscale
2420         c->hcscale_fast(c, dst, dstWidth, src1, src2, srcW, xInc);
2421     }
2422
2423     if (c->chrConvertRange)
2424         c->chrConvertRange(dst, dstWidth);
2425 }
2426
2427 #define DEBUG_SWSCALE_BUFFERS 0
2428 #define DEBUG_BUFFERS(...) if (DEBUG_SWSCALE_BUFFERS) av_log(c, AV_LOG_DEBUG, __VA_ARGS__)
2429
2430 static int RENAME(swScale)(SwsContext *c, const uint8_t* src[], int srcStride[], int srcSliceY,
2431                            int srcSliceH, uint8_t* dst[], int dstStride[])
2432 {
2433     /* load a few things into local vars to make the code more readable? and faster */
2434     const int srcW= c->srcW;
2435     const int dstW= c->dstW;
2436     const int dstH= c->dstH;
2437     const int chrDstW= c->chrDstW;
2438     const int chrSrcW= c->chrSrcW;
2439     const int lumXInc= c->lumXInc;
2440     const int chrXInc= c->chrXInc;
2441     const enum PixelFormat dstFormat= c->dstFormat;
2442     const int flags= c->flags;
2443     int16_t *vLumFilterPos= c->vLumFilterPos;
2444     int16_t *vChrFilterPos= c->vChrFilterPos;
2445     int16_t *hLumFilterPos= c->hLumFilterPos;
2446     int16_t *hChrFilterPos= c->hChrFilterPos;
2447     int16_t *vLumFilter= c->vLumFilter;
2448     int16_t *vChrFilter= c->vChrFilter;
2449     int16_t *hLumFilter= c->hLumFilter;
2450     int16_t *hChrFilter= c->hChrFilter;
2451     int32_t *lumMmxFilter= c->lumMmxFilter;
2452     int32_t *chrMmxFilter= c->chrMmxFilter;
2453     int32_t av_unused *alpMmxFilter= c->alpMmxFilter;
2454     const int vLumFilterSize= c->vLumFilterSize;
2455     const int vChrFilterSize= c->vChrFilterSize;
2456     const int hLumFilterSize= c->hLumFilterSize;
2457     const int hChrFilterSize= c->hChrFilterSize;
2458     int16_t **lumPixBuf= c->lumPixBuf;
2459     int16_t **chrPixBuf= c->chrPixBuf;
2460     int16_t **alpPixBuf= c->alpPixBuf;
2461     const int vLumBufSize= c->vLumBufSize;
2462     const int vChrBufSize= c->vChrBufSize;
2463     uint8_t *formatConvBuffer= c->formatConvBuffer;
2464     const int chrSrcSliceY= srcSliceY >> c->chrSrcVSubSample;
2465     const int chrSrcSliceH= -((-srcSliceH) >> c->chrSrcVSubSample);
2466     int lastDstY;
2467     uint32_t *pal=c->pal_yuv;
2468
2469     /* vars which will change and which we need to store back in the context */
2470     int dstY= c->dstY;
2471     int lumBufIndex= c->lumBufIndex;
2472     int chrBufIndex= c->chrBufIndex;
2473     int lastInLumBuf= c->lastInLumBuf;
2474     int lastInChrBuf= c->lastInChrBuf;
2475
2476     if (isPacked(c->srcFormat)) {
2477         src[0]=
2478         src[1]=
2479         src[2]=
2480         src[3]= src[0];
2481         srcStride[0]=
2482         srcStride[1]=
2483         srcStride[2]=
2484         srcStride[3]= srcStride[0];
2485     }
2486     srcStride[1]<<= c->vChrDrop;
2487     srcStride[2]<<= c->vChrDrop;
2488
2489     DEBUG_BUFFERS("swScale() %p[%d] %p[%d] %p[%d] %p[%d] -> %p[%d] %p[%d] %p[%d] %p[%d]\n",
2490                   src[0], srcStride[0], src[1], srcStride[1], src[2], srcStride[2], src[3], srcStride[3],
2491                   dst[0], dstStride[0], dst[1], dstStride[1], dst[2], dstStride[2], dst[3], dstStride[3]);
2492     DEBUG_BUFFERS("srcSliceY: %d srcSliceH: %d dstY: %d dstH: %d\n",
2493                    srcSliceY,    srcSliceH,    dstY,    dstH);
2494     DEBUG_BUFFERS("vLumFilterSize: %d vLumBufSize: %d vChrFilterSize: %d vChrBufSize: %d\n",
2495                    vLumFilterSize,    vLumBufSize,    vChrFilterSize,    vChrBufSize);
2496
2497     if (dstStride[0]%8 !=0 || dstStride[1]%8 !=0 || dstStride[2]%8 !=0 || dstStride[3]%8 != 0) {
2498         static int warnedAlready=0; //FIXME move this into the context perhaps
2499         if (flags & SWS_PRINT_INFO && !warnedAlready) {
2500             av_log(c, AV_LOG_WARNING, "Warning: dstStride is not aligned!\n"
2501                    "         ->cannot do aligned memory accesses anymore\n");
2502             warnedAlready=1;
2503         }
2504     }
2505
2506     /* Note the user might start scaling the picture in the middle so this
2507        will not get executed. This is not really intended but works
2508        currently, so people might do it. */
2509     if (srcSliceY ==0) {
2510         lumBufIndex=-1;
2511         chrBufIndex=-1;
2512         dstY=0;
2513         lastInLumBuf= -1;
2514         lastInChrBuf= -1;
2515     }
2516
2517     lastDstY= dstY;
2518
2519     for (;dstY < dstH; dstY++) {
2520         unsigned char *dest =dst[0]+dstStride[0]*dstY;
2521         const int chrDstY= dstY>>c->chrDstVSubSample;
2522         unsigned char *uDest=dst[1]+dstStride[1]*chrDstY;
2523         unsigned char *vDest=dst[2]+dstStride[2]*chrDstY;
2524         unsigned char *aDest=(CONFIG_SWSCALE_ALPHA && alpPixBuf) ? dst[3]+dstStride[3]*dstY : NULL;
2525
2526         const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input
2527         const int firstLumSrcY2= vLumFilterPos[FFMIN(dstY | ((1<<c->chrDstVSubSample) - 1), dstH-1)];
2528         const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input
2529         int lastLumSrcY= firstLumSrcY + vLumFilterSize -1; // Last line needed as input
2530         int lastLumSrcY2=firstLumSrcY2+ vLumFilterSize -1; // Last line needed as input
2531         int lastChrSrcY= firstChrSrcY + vChrFilterSize -1; // Last line needed as input
2532         int enough_lines;
2533
2534         //handle holes (FAST_BILINEAR & weird filters)
2535         if (firstLumSrcY > lastInLumBuf) lastInLumBuf= firstLumSrcY-1;
2536         if (firstChrSrcY > lastInChrBuf) lastInChrBuf= firstChrSrcY-1;
2537         assert(firstLumSrcY >= lastInLumBuf - vLumBufSize + 1);
2538         assert(firstChrSrcY >= lastInChrBuf - vChrBufSize + 1);
2539
2540         DEBUG_BUFFERS("dstY: %d\n", dstY);
2541         DEBUG_BUFFERS("\tfirstLumSrcY: %d lastLumSrcY: %d lastInLumBuf: %d\n",
2542                          firstLumSrcY,    lastLumSrcY,    lastInLumBuf);
2543         DEBUG_BUFFERS("\tfirstChrSrcY: %d lastChrSrcY: %d lastInChrBuf: %d\n",
2544                          firstChrSrcY,    lastChrSrcY,    lastInChrBuf);
2545
2546         // Do we have enough lines in this slice to output the dstY line
2547         enough_lines = lastLumSrcY2 < srcSliceY + srcSliceH && lastChrSrcY < -((-srcSliceY - srcSliceH)>>c->chrSrcVSubSample);
2548
2549         if (!enough_lines) {
2550             lastLumSrcY = srcSliceY + srcSliceH - 1;
2551             lastChrSrcY = chrSrcSliceY + chrSrcSliceH - 1;
2552             DEBUG_BUFFERS("buffering slice: lastLumSrcY %d lastChrSrcY %d\n",
2553                                             lastLumSrcY, lastChrSrcY);
2554         }
2555
2556         //Do horizontal scaling
2557         while(lastInLumBuf < lastLumSrcY) {
2558             const uint8_t *src1= src[0]+(lastInLumBuf + 1 - srcSliceY)*srcStride[0];
2559             const uint8_t *src2= src[3]+(lastInLumBuf + 1 - srcSliceY)*srcStride[3];
2560             lumBufIndex++;
2561             assert(lumBufIndex < 2*vLumBufSize);
2562             assert(lastInLumBuf + 1 - srcSliceY < srcSliceH);
2563             assert(lastInLumBuf + 1 - srcSliceY >= 0);
2564             RENAME(hyscale)(c, lumPixBuf[ lumBufIndex ], dstW, src1, srcW, lumXInc,
2565                             hLumFilter, hLumFilterPos, hLumFilterSize,
2566                             formatConvBuffer,
2567                             pal, 0);
2568             if (CONFIG_SWSCALE_ALPHA && alpPixBuf)
2569                 RENAME(hyscale)(c, alpPixBuf[ lumBufIndex ], dstW, src2, srcW, lumXInc,
2570                                 hLumFilter, hLumFilterPos, hLumFilterSize,
2571                                 formatConvBuffer,
2572                                 pal, 1);
2573             lastInLumBuf++;
2574             DEBUG_BUFFERS("\t\tlumBufIndex %d: lastInLumBuf: %d\n",
2575                                lumBufIndex,    lastInLumBuf);
2576         }
2577         while(lastInChrBuf < lastChrSrcY) {
2578             const uint8_t *src1= src[1]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[1];
2579             const uint8_t *src2= src[2]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[2];
2580             chrBufIndex++;
2581             assert(chrBufIndex < 2*vChrBufSize);
2582             assert(lastInChrBuf + 1 - chrSrcSliceY < (chrSrcSliceH));
2583             assert(lastInChrBuf + 1 - chrSrcSliceY >= 0);
2584             //FIXME replace parameters through context struct (some at least)
2585
2586             if (c->needs_hcscale)
2587                 RENAME(hcscale)(c, chrPixBuf[ chrBufIndex ], chrDstW, src1, src2, chrSrcW, chrXInc,
2588                                 hChrFilter, hChrFilterPos, hChrFilterSize,
2589                                 formatConvBuffer,
2590                                 pal);
2591             lastInChrBuf++;
2592             DEBUG_BUFFERS("\t\tchrBufIndex %d: lastInChrBuf: %d\n",
2593                                chrBufIndex,    lastInChrBuf);
2594         }
2595         //wrap buf index around to stay inside the ring buffer
2596         if (lumBufIndex >= vLumBufSize) lumBufIndex-= vLumBufSize;
2597         if (chrBufIndex >= vChrBufSize) chrBufIndex-= vChrBufSize;
2598         if (!enough_lines)
2599             break; //we can't output a dstY line so let's try with the next slice
2600
2601         c->blueDither= ff_dither8[dstY&1];
2602         if (c->dstFormat == PIX_FMT_RGB555 || c->dstFormat == PIX_FMT_BGR555)
2603             c->greenDither= ff_dither8[dstY&1];
2604         else
2605             c->greenDither= ff_dither4[dstY&1];
2606         c->redDither= ff_dither8[(dstY+1)&1];
2607         if (dstY < dstH-2) {
2608             const int16_t **lumSrcPtr= (const int16_t **) lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2609             const int16_t **chrSrcPtr= (const int16_t **) chrPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2610             const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;
2611             int i;
2612             if (flags & SWS_ACCURATE_RND) {
2613                 int s= APCK_SIZE / 8;
2614                 for (i=0; i<vLumFilterSize; i+=2) {
2615                     *(const void**)&lumMmxFilter[s*i              ]= lumSrcPtr[i  ];
2616                     *(const void**)&lumMmxFilter[s*i+APCK_PTR2/4  ]= lumSrcPtr[i+(vLumFilterSize>1)];
2617                               lumMmxFilter[s*i+APCK_COEF/4  ]=
2618                               lumMmxFilter[s*i+APCK_COEF/4+1]= vLumFilter[dstY*vLumFilterSize + i    ]
2619                         + (vLumFilterSize>1 ? vLumFilter[dstY*vLumFilterSize + i + 1]<<16 : 0);
2620                     if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2621                         *(const void**)&alpMmxFilter[s*i              ]= alpSrcPtr[i  ];
2622                         *(const void**)&alpMmxFilter[s*i+APCK_PTR2/4  ]= alpSrcPtr[i+(vLumFilterSize>1)];
2623                                   alpMmxFilter[s*i+APCK_COEF/4  ]=
2624                                   alpMmxFilter[s*i+APCK_COEF/4+1]= lumMmxFilter[s*i+APCK_COEF/4  ];
2625                     }
2626                 }
2627                 for (i=0; i<vChrFilterSize; i+=2) {
2628                     *(const void**)&chrMmxFilter[s*i              ]= chrSrcPtr[i  ];
2629                     *(const void**)&chrMmxFilter[s*i+APCK_PTR2/4  ]= chrSrcPtr[i+(vChrFilterSize>1)];
2630                               chrMmxFilter[s*i+APCK_COEF/4  ]=
2631                               chrMmxFilter[s*i+APCK_COEF/4+1]= vChrFilter[chrDstY*vChrFilterSize + i    ]
2632                         + (vChrFilterSize>1 ? vChrFilter[chrDstY*vChrFilterSize + i + 1]<<16 : 0);
2633                 }
2634             } else {
2635                 for (i=0; i<vLumFilterSize; i++) {
2636                     lumMmxFilter[4*i+0]= (int32_t)lumSrcPtr[i];
2637                     lumMmxFilter[4*i+1]= (uint64_t)lumSrcPtr[i] >> 32;
2638                     lumMmxFilter[4*i+2]=
2639                     lumMmxFilter[4*i+3]=
2640                         ((uint16_t)vLumFilter[dstY*vLumFilterSize + i])*0x10001;
2641                     if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2642                         alpMmxFilter[4*i+0]= (int32_t)alpSrcPtr[i];
2643                         alpMmxFilter[4*i+1]= (uint64_t)alpSrcPtr[i] >> 32;
2644                         alpMmxFilter[4*i+2]=
2645                         alpMmxFilter[4*i+3]= lumMmxFilter[4*i+2];
2646                     }
2647                 }
2648                 for (i=0; i<vChrFilterSize; i++) {
2649                     chrMmxFilter[4*i+0]= (int32_t)chrSrcPtr[i];
2650                     chrMmxFilter[4*i+1]= (uint64_t)chrSrcPtr[i] >> 32;
2651                     chrMmxFilter[4*i+2]=
2652                     chrMmxFilter[4*i+3]=
2653                         ((uint16_t)vChrFilter[chrDstY*vChrFilterSize + i])*0x10001;
2654                 }
2655             }
2656             if (dstFormat == PIX_FMT_NV12 || dstFormat == PIX_FMT_NV21) {
2657                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2658                 if (dstY&chrSkipMask) uDest= NULL; //FIXME split functions in lumi / chromi
2659                 c->yuv2nv12X(c,
2660                              vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2661                              vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2662                              dest, uDest, dstW, chrDstW, dstFormat);
2663             } else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12 like
2664                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2665                 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
2666                 if (is16BPS(dstFormat)) {
2667                     yuv2yuvX16inC(
2668                                   vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2669                                   vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2670                                   alpSrcPtr, (uint16_t *) dest, (uint16_t *) uDest, (uint16_t *) vDest, (uint16_t *) aDest, dstW, chrDstW,
2671                                   dstFormat);
2672                 } else if (vLumFilterSize == 1 && vChrFilterSize == 1) { // unscaled YV12
2673                     const int16_t *lumBuf = lumSrcPtr[0];
2674                     const int16_t *chrBuf= chrSrcPtr[0];
2675                     const int16_t *alpBuf= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? alpSrcPtr[0] : NULL;
2676                     c->yuv2yuv1(c, lumBuf, chrBuf, alpBuf, dest, uDest, vDest, aDest, dstW, chrDstW);
2677                 } else { //General YV12
2678                     c->yuv2yuvX(c,
2679                                 vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2680                                 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2681                                 alpSrcPtr, dest, uDest, vDest, aDest, dstW, chrDstW);
2682                 }
2683             } else {
2684                 assert(lumSrcPtr + vLumFilterSize - 1 < lumPixBuf + vLumBufSize*2);
2685                 assert(chrSrcPtr + vChrFilterSize - 1 < chrPixBuf + vChrBufSize*2);
2686                 if (vLumFilterSize == 1 && vChrFilterSize == 2) { //unscaled RGB
2687                     int chrAlpha= vChrFilter[2*dstY+1];
2688                     if(flags & SWS_FULL_CHR_H_INT) {
2689                         yuv2rgbXinC_full(c, //FIXME write a packed1_full function
2690                                          vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2691                                          vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2692                                          alpSrcPtr, dest, dstW, dstY);
2693                     } else {
2694                         c->yuv2packed1(c, *lumSrcPtr, *chrSrcPtr, *(chrSrcPtr+1),
2695                                        alpPixBuf ? *alpSrcPtr : NULL,
2696                                        dest, dstW, chrAlpha, dstFormat, flags, dstY);
2697                     }
2698                 } else if (vLumFilterSize == 2 && vChrFilterSize == 2) { //bilinear upscale RGB
2699                     int lumAlpha= vLumFilter[2*dstY+1];
2700                     int chrAlpha= vChrFilter[2*dstY+1];
2701                     lumMmxFilter[2]=
2702                     lumMmxFilter[3]= vLumFilter[2*dstY   ]*0x10001;
2703                     chrMmxFilter[2]=
2704                     chrMmxFilter[3]= vChrFilter[2*chrDstY]*0x10001;
2705                     if(flags & SWS_FULL_CHR_H_INT) {
2706                         yuv2rgbXinC_full(c, //FIXME write a packed2_full function
2707                                          vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2708                                          vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2709                                          alpSrcPtr, dest, dstW, dstY);
2710                     } else {
2711                         c->yuv2packed2(c, *lumSrcPtr, *(lumSrcPtr+1), *chrSrcPtr, *(chrSrcPtr+1),
2712                                        alpPixBuf ? *alpSrcPtr : NULL, alpPixBuf ? *(alpSrcPtr+1) : NULL,
2713                                        dest, dstW, lumAlpha, chrAlpha, dstY);
2714                     }
2715                 } else { //general RGB
2716                     if(flags & SWS_FULL_CHR_H_INT) {
2717                         yuv2rgbXinC_full(c,
2718                                          vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2719                                          vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2720                                          alpSrcPtr, dest, dstW, dstY);
2721                     } else {
2722                         c->yuv2packedX(c,
2723                                        vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2724                                        vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2725                                        alpSrcPtr, dest, dstW, dstY);
2726                     }
2727                 }
2728             }
2729         } else { // hmm looks like we can't use MMX here without overwriting this array's tail
2730             const int16_t **lumSrcPtr= (const int16_t **)lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2731             const int16_t **chrSrcPtr= (const int16_t **)chrPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2732             const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **)alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;
2733             if (dstFormat == PIX_FMT_NV12 || dstFormat == PIX_FMT_NV21) {
2734                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2735                 if (dstY&chrSkipMask) uDest= NULL; //FIXME split functions in lumi / chromi
2736                 yuv2nv12XinC(
2737                              vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2738                              vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2739                              dest, uDest, dstW, chrDstW, dstFormat);
2740             } else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12
2741                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2742                 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
2743                 if (is16BPS(dstFormat)) {
2744                     yuv2yuvX16inC(
2745                                   vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2746                                   vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2747                                   alpSrcPtr, (uint16_t *) dest, (uint16_t *) uDest, (uint16_t *) vDest, (uint16_t *) aDest, dstW, chrDstW,
2748                                   dstFormat);
2749                 } else {
2750                     yuv2yuvXinC(
2751                                 vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2752                                 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2753                                 alpSrcPtr, dest, uDest, vDest, aDest, dstW, chrDstW);
2754                 }
2755             } else {
2756                 assert(lumSrcPtr + vLumFilterSize - 1 < lumPixBuf + vLumBufSize*2);
2757                 assert(chrSrcPtr + vChrFilterSize - 1 < chrPixBuf + vChrBufSize*2);
2758                 if(flags & SWS_FULL_CHR_H_INT) {
2759                     yuv2rgbXinC_full(c,
2760                                      vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2761                                      vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2762                                      alpSrcPtr, dest, dstW, dstY);
2763                 } else {
2764                     yuv2packedXinC(c,
2765                                    vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2766                                    vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2767                                    alpSrcPtr, dest, dstW, dstY);
2768                 }
2769             }
2770         }
2771     }
2772
2773     if ((dstFormat == PIX_FMT_YUVA420P) && !alpPixBuf)
2774         fillPlane(dst[3], dstStride[3], dstW, dstY-lastDstY, lastDstY, 255);
2775
2776     if (flags & SWS_CPU_CAPS_MMX2 )  __asm__ volatile("sfence":::"memory");
2777     /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */
2778     if (flags & SWS_CPU_CAPS_3DNOW)  __asm__ volatile("femms" :::"memory");
2779     else                             __asm__ volatile("emms"  :::"memory");
2780     /* store changed local vars back in the context */
2781     c->dstY= dstY;
2782     c->lumBufIndex= lumBufIndex;
2783     c->chrBufIndex= chrBufIndex;
2784     c->lastInLumBuf= lastInLumBuf;
2785     c->lastInChrBuf= lastInChrBuf;
2786
2787     return dstY - lastDstY;
2788 }
2789
2790 static void RENAME(sws_init_swScale)(SwsContext *c)
2791 {
2792     enum PixelFormat srcFormat = c->srcFormat;
2793
2794     c->yuv2nv12X    = RENAME(yuv2nv12X   );
2795     c->yuv2yuv1     = RENAME(yuv2yuv1    );
2796     c->yuv2yuvX     = RENAME(yuv2yuvX    );
2797     c->yuv2packed1  = RENAME(yuv2packed1 );
2798     c->yuv2packed2  = RENAME(yuv2packed2 );
2799     c->yuv2packedX  = RENAME(yuv2packedX );
2800
2801     c->hScale       = RENAME(hScale      );
2802
2803     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2804     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
2805     {
2806         c->hyscale_fast = RENAME(hyscale_fast);
2807         c->hcscale_fast = RENAME(hcscale_fast);
2808     } else {
2809         c->hyscale_fast = NULL;
2810         c->hcscale_fast = NULL;
2811     }
2812
2813     switch(srcFormat) {
2814         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;
2815         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;
2816         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;
2817         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;
2818         case PIX_FMT_YUV420P16BE:
2819         case PIX_FMT_YUV422P16BE:
2820         case PIX_FMT_YUV444P16BE: c->chrToYV12 = RENAME(BEToUV); break;
2821         case PIX_FMT_YUV420P16LE:
2822         case PIX_FMT_YUV422P16LE:
2823         case PIX_FMT_YUV444P16LE: c->chrToYV12 = RENAME(LEToUV); break;
2824         default: break;
2825     }
2826     if (c->chrSrcHSubSample) {
2827         switch(srcFormat) {
2828         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV_half); break;
2829         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV_half); break;
2830         default: break;
2831         }
2832     } else {
2833         switch(srcFormat) {
2834         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
2835         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
2836         default: break;
2837         }
2838     }
2839
2840     switch (srcFormat) {
2841     case PIX_FMT_YUYV422  :
2842     case PIX_FMT_YUV420P16BE:
2843     case PIX_FMT_YUV422P16BE:
2844     case PIX_FMT_YUV444P16BE:
2845     case PIX_FMT_Y400A    :
2846     case PIX_FMT_GRAY16BE : c->lumToYV12 = RENAME(yuy2ToY); break;
2847     case PIX_FMT_UYVY422  :
2848     case PIX_FMT_YUV420P16LE:
2849     case PIX_FMT_YUV422P16LE:
2850     case PIX_FMT_YUV444P16LE:
2851     case PIX_FMT_GRAY16LE : c->lumToYV12 = RENAME(uyvyToY); break;
2852     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
2853     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
2854     default: break;
2855     }
2856     if (c->alpPixBuf) {
2857         switch (srcFormat) {
2858         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;
2859         default: break;
2860         }
2861     }
2862
2863     if (c->srcRange != c->dstRange && !isAnyRGB(c->dstFormat)) {
2864         if (c->srcRange) {
2865             c->lumConvertRange = RENAME(lumRangeFromJpeg);
2866             c->chrConvertRange = RENAME(chrRangeFromJpeg);
2867         } else {
2868             c->lumConvertRange = RENAME(lumRangeToJpeg);
2869             c->chrConvertRange = RENAME(chrRangeToJpeg);
2870         }
2871     }
2872 }