swscale: remove duplicated x86/ functions.
[ffmpeg.git] / libswscale / x86 / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of Libav.
5  *
6  * Libav is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * Libav is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with Libav; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #include "swscale_template.h"
22
23 #undef REAL_MOVNTQ
24 #undef MOVNTQ
25 #undef PAVGB
26 #undef PREFETCH
27
28 #if COMPILE_TEMPLATE_AMD3DNOW
29 #define PREFETCH  "prefetch"
30 #elif COMPILE_TEMPLATE_MMX2
31 #define PREFETCH "prefetchnta"
32 #else
33 #define PREFETCH  " # nop"
34 #endif
35
36 #if COMPILE_TEMPLATE_MMX2
37 #define PAVGB(a,b) "pavgb " #a ", " #b " \n\t"
38 #elif COMPILE_TEMPLATE_AMD3DNOW
39 #define PAVGB(a,b) "pavgusb " #a ", " #b " \n\t"
40 #endif
41
42 #if COMPILE_TEMPLATE_MMX2
43 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
44 #else
45 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
46 #endif
47 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
48
49 #define YSCALEYUV2YV12X(x, offset, dest, width) \
50     __asm__ volatile(\
51         "xor                          %%"REG_a", %%"REG_a"  \n\t"\
52         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
53         "movq                             %%mm3, %%mm4      \n\t"\
54         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
55         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
56         ".p2align                             4             \n\t" /* FIXME Unroll? */\
57         "1:                                                 \n\t"\
58         "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\
59         "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm2      \n\t" /* srcData */\
60         "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm5      \n\t" /* srcData */\
61         "add                                $16, %%"REG_d"  \n\t"\
62         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
63         "test                         %%"REG_S", %%"REG_S"  \n\t"\
64         "pmulhw                           %%mm0, %%mm2      \n\t"\
65         "pmulhw                           %%mm0, %%mm5      \n\t"\
66         "paddw                            %%mm2, %%mm3      \n\t"\
67         "paddw                            %%mm5, %%mm4      \n\t"\
68         " jnz                                1b             \n\t"\
69         "psraw                               $3, %%mm3      \n\t"\
70         "psraw                               $3, %%mm4      \n\t"\
71         "packuswb                         %%mm4, %%mm3      \n\t"\
72         MOVNTQ(%%mm3, (%1, %%REGa))\
73         "add                                 $8, %%"REG_a"  \n\t"\
74         "cmp                                 %2, %%"REG_a"  \n\t"\
75         "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
76         "movq                             %%mm3, %%mm4      \n\t"\
77         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
78         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
79         "jb                                  1b             \n\t"\
80         :: "r" (&c->redDither),\
81         "r" (dest), "g" ((x86_reg)width)\
82         : "%"REG_a, "%"REG_d, "%"REG_S\
83     );
84
85 #define YSCALEYUV2YV12X_ACCURATE(x, offset, dest, width) \
86     __asm__ volatile(\
87         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
88         "xor                          %%"REG_a", %%"REG_a"  \n\t"\
89         "pxor                             %%mm4, %%mm4      \n\t"\
90         "pxor                             %%mm5, %%mm5      \n\t"\
91         "pxor                             %%mm6, %%mm6      \n\t"\
92         "pxor                             %%mm7, %%mm7      \n\t"\
93         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
94         ".p2align                             4             \n\t"\
95         "1:                                                 \n\t"\
96         "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm0      \n\t" /* srcData */\
97         "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm2      \n\t" /* srcData */\
98         "mov        "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"  \n\t"\
99         "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm1      \n\t" /* srcData */\
100         "movq                             %%mm0, %%mm3      \n\t"\
101         "punpcklwd                        %%mm1, %%mm0      \n\t"\
102         "punpckhwd                        %%mm1, %%mm3      \n\t"\
103         "movq       "STR(APCK_COEF)"(%%"REG_d"), %%mm1      \n\t" /* filterCoeff */\
104         "pmaddwd                          %%mm1, %%mm0      \n\t"\
105         "pmaddwd                          %%mm1, %%mm3      \n\t"\
106         "paddd                            %%mm0, %%mm4      \n\t"\
107         "paddd                            %%mm3, %%mm5      \n\t"\
108         "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm3      \n\t" /* srcData */\
109         "mov        "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"  \n\t"\
110         "add                  $"STR(APCK_SIZE)", %%"REG_d"  \n\t"\
111         "test                         %%"REG_S", %%"REG_S"  \n\t"\
112         "movq                             %%mm2, %%mm0      \n\t"\
113         "punpcklwd                        %%mm3, %%mm2      \n\t"\
114         "punpckhwd                        %%mm3, %%mm0      \n\t"\
115         "pmaddwd                          %%mm1, %%mm2      \n\t"\
116         "pmaddwd                          %%mm1, %%mm0      \n\t"\
117         "paddd                            %%mm2, %%mm6      \n\t"\
118         "paddd                            %%mm0, %%mm7      \n\t"\
119         " jnz                                1b             \n\t"\
120         "psrad                              $16, %%mm4      \n\t"\
121         "psrad                              $16, %%mm5      \n\t"\
122         "psrad                              $16, %%mm6      \n\t"\
123         "psrad                              $16, %%mm7      \n\t"\
124         "movq             "VROUNDER_OFFSET"(%0), %%mm0      \n\t"\
125         "packssdw                         %%mm5, %%mm4      \n\t"\
126         "packssdw                         %%mm7, %%mm6      \n\t"\
127         "paddw                            %%mm0, %%mm4      \n\t"\
128         "paddw                            %%mm0, %%mm6      \n\t"\
129         "psraw                               $3, %%mm4      \n\t"\
130         "psraw                               $3, %%mm6      \n\t"\
131         "packuswb                         %%mm6, %%mm4      \n\t"\
132         MOVNTQ(%%mm4, (%1, %%REGa))\
133         "add                                 $8, %%"REG_a"  \n\t"\
134         "cmp                                 %2, %%"REG_a"  \n\t"\
135         "lea                     " offset "(%0), %%"REG_d"  \n\t"\
136         "pxor                             %%mm4, %%mm4      \n\t"\
137         "pxor                             %%mm5, %%mm5      \n\t"\
138         "pxor                             %%mm6, %%mm6      \n\t"\
139         "pxor                             %%mm7, %%mm7      \n\t"\
140         "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
141         "jb                                  1b             \n\t"\
142         :: "r" (&c->redDither),\
143         "r" (dest), "g" ((x86_reg)width)\
144         : "%"REG_a, "%"REG_d, "%"REG_S\
145     );
146
147 #define YSCALEYUV2YV121 \
148     "mov %2, %%"REG_a"                    \n\t"\
149     ".p2align               4             \n\t" /* FIXME Unroll? */\
150     "1:                                   \n\t"\
151     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
152     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
153     "psraw                 $7, %%mm0      \n\t"\
154     "psraw                 $7, %%mm1      \n\t"\
155     "packuswb           %%mm1, %%mm0      \n\t"\
156     MOVNTQ(%%mm0, (%1, %%REGa))\
157     "add                   $8, %%"REG_a"  \n\t"\
158     "jnc                   1b             \n\t"
159
160 #define YSCALEYUV2YV121_ACCURATE \
161     "mov %2, %%"REG_a"                    \n\t"\
162     "pcmpeqw %%mm7, %%mm7                 \n\t"\
163     "psrlw                 $15, %%mm7     \n\t"\
164     "psllw                  $6, %%mm7     \n\t"\
165     ".p2align                4            \n\t" /* FIXME Unroll? */\
166     "1:                                   \n\t"\
167     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
168     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
169     "paddsw             %%mm7, %%mm0      \n\t"\
170     "paddsw             %%mm7, %%mm1      \n\t"\
171     "psraw                 $7, %%mm0      \n\t"\
172     "psraw                 $7, %%mm1      \n\t"\
173     "packuswb           %%mm1, %%mm0      \n\t"\
174     MOVNTQ(%%mm0, (%1, %%REGa))\
175     "add                   $8, %%"REG_a"  \n\t"\
176     "jnc                   1b             \n\t"
177
178 /*
179     :: "m" (-lumFilterSize), "m" (-chrFilterSize),
180        "m" (lumMmxFilter+lumFilterSize*4), "m" (chrMmxFilter+chrFilterSize*4),
181        "r" (dest), "m" (dstW_reg),
182        "m" (lumSrc+lumFilterSize), "m" (chrSrc+chrFilterSize)
183     : "%eax", "%ebx", "%ecx", "%edx", "%esi"
184 */
185 #define YSCALEYUV2PACKEDX_UV \
186     __asm__ volatile(\
187         "xor                   %%"REG_a", %%"REG_a"     \n\t"\
188         ".p2align                      4                \n\t"\
189         "nop                                            \n\t"\
190         "1:                                             \n\t"\
191         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
192         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
193         "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
194         "movq                      %%mm3, %%mm4         \n\t"\
195         ".p2align                      4                \n\t"\
196         "2:                                             \n\t"\
197         "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
198         "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
199         "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
200         "add                         $16, %%"REG_d"     \n\t"\
201         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
202         "pmulhw                    %%mm0, %%mm2         \n\t"\
203         "pmulhw                    %%mm0, %%mm5         \n\t"\
204         "paddw                     %%mm2, %%mm3         \n\t"\
205         "paddw                     %%mm5, %%mm4         \n\t"\
206         "test                  %%"REG_S", %%"REG_S"     \n\t"\
207         " jnz                         2b                \n\t"\
208
209 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
210     "lea                "offset"(%0), %%"REG_d"     \n\t"\
211     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
212     "movq      "VROUNDER_OFFSET"(%0), "#dst1"       \n\t"\
213     "movq                    "#dst1", "#dst2"       \n\t"\
214     ".p2align                      4                \n\t"\
215     "2:                                             \n\t"\
216     "movq               8(%%"REG_d"), "#coeff"      \n\t" /* filterCoeff */\
217     "movq  (%%"REG_S", %%"REG_a", 2), "#src1"       \n\t" /* Y1srcData */\
218     "movq 8(%%"REG_S", %%"REG_a", 2), "#src2"       \n\t" /* Y2srcData */\
219     "add                         $16, %%"REG_d"            \n\t"\
220     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
221     "pmulhw                 "#coeff", "#src1"       \n\t"\
222     "pmulhw                 "#coeff", "#src2"       \n\t"\
223     "paddw                   "#src1", "#dst1"       \n\t"\
224     "paddw                   "#src2", "#dst2"       \n\t"\
225     "test                  %%"REG_S", %%"REG_S"     \n\t"\
226     " jnz                         2b                \n\t"\
227
228 #define YSCALEYUV2PACKEDX \
229     YSCALEYUV2PACKEDX_UV \
230     YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
231
232 #define YSCALEYUV2PACKEDX_END                     \
233         :: "r" (&c->redDither),                   \
234             "m" (dummy), "m" (dummy), "m" (dummy),\
235             "r" (dest), "m" (dstW_reg)            \
236         : "%"REG_a, "%"REG_d, "%"REG_S            \
237     );
238
239 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
240     __asm__ volatile(\
241         "xor %%"REG_a", %%"REG_a"                       \n\t"\
242         ".p2align                      4                \n\t"\
243         "nop                                            \n\t"\
244         "1:                                             \n\t"\
245         "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
246         "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
247         "pxor                      %%mm4, %%mm4         \n\t"\
248         "pxor                      %%mm5, %%mm5         \n\t"\
249         "pxor                      %%mm6, %%mm6         \n\t"\
250         "pxor                      %%mm7, %%mm7         \n\t"\
251         ".p2align                      4                \n\t"\
252         "2:                                             \n\t"\
253         "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
254         "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
255         "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
256         "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
257         "movq                      %%mm0, %%mm3         \n\t"\
258         "punpcklwd                 %%mm1, %%mm0         \n\t"\
259         "punpckhwd                 %%mm1, %%mm3         \n\t"\
260         "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
261         "pmaddwd                   %%mm1, %%mm0         \n\t"\
262         "pmaddwd                   %%mm1, %%mm3         \n\t"\
263         "paddd                     %%mm0, %%mm4         \n\t"\
264         "paddd                     %%mm3, %%mm5         \n\t"\
265         "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
266         "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
267         "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
268         "test                  %%"REG_S", %%"REG_S"     \n\t"\
269         "movq                      %%mm2, %%mm0         \n\t"\
270         "punpcklwd                 %%mm3, %%mm2         \n\t"\
271         "punpckhwd                 %%mm3, %%mm0         \n\t"\
272         "pmaddwd                   %%mm1, %%mm2         \n\t"\
273         "pmaddwd                   %%mm1, %%mm0         \n\t"\
274         "paddd                     %%mm2, %%mm6         \n\t"\
275         "paddd                     %%mm0, %%mm7         \n\t"\
276         " jnz                         2b                \n\t"\
277         "psrad                       $16, %%mm4         \n\t"\
278         "psrad                       $16, %%mm5         \n\t"\
279         "psrad                       $16, %%mm6         \n\t"\
280         "psrad                       $16, %%mm7         \n\t"\
281         "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
282         "packssdw                  %%mm5, %%mm4         \n\t"\
283         "packssdw                  %%mm7, %%mm6         \n\t"\
284         "paddw                     %%mm0, %%mm4         \n\t"\
285         "paddw                     %%mm0, %%mm6         \n\t"\
286         "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
287         "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
288
289 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
290     "lea                "offset"(%0), %%"REG_d"     \n\t"\
291     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
292     "pxor                      %%mm1, %%mm1         \n\t"\
293     "pxor                      %%mm5, %%mm5         \n\t"\
294     "pxor                      %%mm7, %%mm7         \n\t"\
295     "pxor                      %%mm6, %%mm6         \n\t"\
296     ".p2align                      4                \n\t"\
297     "2:                                             \n\t"\
298     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
299     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
300     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
301     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
302     "movq                      %%mm0, %%mm3         \n\t"\
303     "punpcklwd                 %%mm4, %%mm0         \n\t"\
304     "punpckhwd                 %%mm4, %%mm3         \n\t"\
305     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
306     "pmaddwd                   %%mm4, %%mm0         \n\t"\
307     "pmaddwd                   %%mm4, %%mm3         \n\t"\
308     "paddd                     %%mm0, %%mm1         \n\t"\
309     "paddd                     %%mm3, %%mm5         \n\t"\
310     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
311     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
312     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
313     "test                  %%"REG_S", %%"REG_S"     \n\t"\
314     "movq                      %%mm2, %%mm0         \n\t"\
315     "punpcklwd                 %%mm3, %%mm2         \n\t"\
316     "punpckhwd                 %%mm3, %%mm0         \n\t"\
317     "pmaddwd                   %%mm4, %%mm2         \n\t"\
318     "pmaddwd                   %%mm4, %%mm0         \n\t"\
319     "paddd                     %%mm2, %%mm7         \n\t"\
320     "paddd                     %%mm0, %%mm6         \n\t"\
321     " jnz                         2b                \n\t"\
322     "psrad                       $16, %%mm1         \n\t"\
323     "psrad                       $16, %%mm5         \n\t"\
324     "psrad                       $16, %%mm7         \n\t"\
325     "psrad                       $16, %%mm6         \n\t"\
326     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
327     "packssdw                  %%mm5, %%mm1         \n\t"\
328     "packssdw                  %%mm6, %%mm7         \n\t"\
329     "paddw                     %%mm0, %%mm1         \n\t"\
330     "paddw                     %%mm0, %%mm7         \n\t"\
331     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
332     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
333
334 #define YSCALEYUV2PACKEDX_ACCURATE \
335     YSCALEYUV2PACKEDX_ACCURATE_UV \
336     YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
337
338 #define YSCALEYUV2RGBX \
339     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
340     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
341     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
342     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
343     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
344     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
345     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
346     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
347     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
348     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
349     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
350     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
351     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
352     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
353     "paddw           %%mm3, %%mm4       \n\t"\
354     "movq            %%mm2, %%mm0       \n\t"\
355     "movq            %%mm5, %%mm6       \n\t"\
356     "movq            %%mm4, %%mm3       \n\t"\
357     "punpcklwd       %%mm2, %%mm2       \n\t"\
358     "punpcklwd       %%mm5, %%mm5       \n\t"\
359     "punpcklwd       %%mm4, %%mm4       \n\t"\
360     "paddw           %%mm1, %%mm2       \n\t"\
361     "paddw           %%mm1, %%mm5       \n\t"\
362     "paddw           %%mm1, %%mm4       \n\t"\
363     "punpckhwd       %%mm0, %%mm0       \n\t"\
364     "punpckhwd       %%mm6, %%mm6       \n\t"\
365     "punpckhwd       %%mm3, %%mm3       \n\t"\
366     "paddw           %%mm7, %%mm0       \n\t"\
367     "paddw           %%mm7, %%mm6       \n\t"\
368     "paddw           %%mm7, %%mm3       \n\t"\
369     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
370     "packuswb        %%mm0, %%mm2       \n\t"\
371     "packuswb        %%mm6, %%mm5       \n\t"\
372     "packuswb        %%mm3, %%mm4       \n\t"\
373
374 #define REAL_YSCALEYUV2PACKED(index, c) \
375     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
376     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
377     "psraw                $3, %%mm0                           \n\t"\
378     "psraw                $3, %%mm1                           \n\t"\
379     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
380     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
381     "xor            "#index", "#index"                        \n\t"\
382     ".p2align              4            \n\t"\
383     "1:                                 \n\t"\
384     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
385     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
386     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
387     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
388     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
389     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
390     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
391     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
392     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
393     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
394     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
395     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
396     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
397     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
398     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
399     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
400     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
401     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
402     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
403     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
404     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
405     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
406     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
407     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
408     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
409
410 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
411
412 #define REAL_YSCALEYUV2RGB_UV(index, c) \
413     "xor            "#index", "#index"  \n\t"\
414     ".p2align              4            \n\t"\
415     "1:                                 \n\t"\
416     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
417     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
418     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
419     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
420     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
421     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
422     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
423     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
424     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
425     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
426     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
427     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
428     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
429     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
430     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
431     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
432     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
433     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
434     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
435     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
436
437 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
438     "movq  ("#b1", "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
439     "movq  ("#b2", "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
440     "movq 8("#b1", "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
441     "movq 8("#b2", "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
442     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
443     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
444     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
445     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
446     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
447     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
448     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
449     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
450
451 #define REAL_YSCALEYUV2RGB_COEFF(c) \
452     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
453     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
454     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
455     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
456     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
457     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
458     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
459     "paddw             %%mm3, %%mm4     \n\t"\
460     "movq              %%mm2, %%mm0     \n\t"\
461     "movq              %%mm5, %%mm6     \n\t"\
462     "movq              %%mm4, %%mm3     \n\t"\
463     "punpcklwd         %%mm2, %%mm2     \n\t"\
464     "punpcklwd         %%mm5, %%mm5     \n\t"\
465     "punpcklwd         %%mm4, %%mm4     \n\t"\
466     "paddw             %%mm1, %%mm2     \n\t"\
467     "paddw             %%mm1, %%mm5     \n\t"\
468     "paddw             %%mm1, %%mm4     \n\t"\
469     "punpckhwd         %%mm0, %%mm0     \n\t"\
470     "punpckhwd         %%mm6, %%mm6     \n\t"\
471     "punpckhwd         %%mm3, %%mm3     \n\t"\
472     "paddw             %%mm7, %%mm0     \n\t"\
473     "paddw             %%mm7, %%mm6     \n\t"\
474     "paddw             %%mm7, %%mm3     \n\t"\
475     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
476     "packuswb          %%mm0, %%mm2     \n\t"\
477     "packuswb          %%mm6, %%mm5     \n\t"\
478     "packuswb          %%mm3, %%mm4     \n\t"\
479
480 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
481
482 #define YSCALEYUV2RGB(index, c) \
483     REAL_YSCALEYUV2RGB_UV(index, c) \
484     REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
485     REAL_YSCALEYUV2RGB_COEFF(c)
486
487 #define REAL_YSCALEYUV2PACKED1(index, c) \
488     "xor            "#index", "#index"  \n\t"\
489     ".p2align              4            \n\t"\
490     "1:                                 \n\t"\
491     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
492     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
493     "psraw                $7, %%mm3     \n\t" \
494     "psraw                $7, %%mm4     \n\t" \
495     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
496     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
497     "psraw                $7, %%mm1     \n\t" \
498     "psraw                $7, %%mm7     \n\t" \
499
500 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
501
502 #define REAL_YSCALEYUV2RGB1(index, c) \
503     "xor            "#index", "#index"  \n\t"\
504     ".p2align              4            \n\t"\
505     "1:                                 \n\t"\
506     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
507     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
508     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
509     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
510     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
511     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
512     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
513     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
514     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
515     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
516     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
517     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
518     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
519     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
520     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
521     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
522     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
523     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
524     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
525     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
526     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
527     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
528     "paddw             %%mm3, %%mm4     \n\t"\
529     "movq              %%mm2, %%mm0     \n\t"\
530     "movq              %%mm5, %%mm6     \n\t"\
531     "movq              %%mm4, %%mm3     \n\t"\
532     "punpcklwd         %%mm2, %%mm2     \n\t"\
533     "punpcklwd         %%mm5, %%mm5     \n\t"\
534     "punpcklwd         %%mm4, %%mm4     \n\t"\
535     "paddw             %%mm1, %%mm2     \n\t"\
536     "paddw             %%mm1, %%mm5     \n\t"\
537     "paddw             %%mm1, %%mm4     \n\t"\
538     "punpckhwd         %%mm0, %%mm0     \n\t"\
539     "punpckhwd         %%mm6, %%mm6     \n\t"\
540     "punpckhwd         %%mm3, %%mm3     \n\t"\
541     "paddw             %%mm7, %%mm0     \n\t"\
542     "paddw             %%mm7, %%mm6     \n\t"\
543     "paddw             %%mm7, %%mm3     \n\t"\
544     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
545     "packuswb          %%mm0, %%mm2     \n\t"\
546     "packuswb          %%mm6, %%mm5     \n\t"\
547     "packuswb          %%mm3, %%mm4     \n\t"\
548
549 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
550
551 #define REAL_YSCALEYUV2PACKED1b(index, c) \
552     "xor "#index", "#index"             \n\t"\
553     ".p2align              4            \n\t"\
554     "1:                                 \n\t"\
555     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
556     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
557     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
558     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
559     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
560     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
561     "psrlw                $8, %%mm3     \n\t" \
562     "psrlw                $8, %%mm4     \n\t" \
563     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
564     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
565     "psraw                $7, %%mm1     \n\t" \
566     "psraw                $7, %%mm7     \n\t"
567 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
568
569 // do vertical chrominance interpolation
570 #define REAL_YSCALEYUV2RGB1b(index, c) \
571     "xor            "#index", "#index"  \n\t"\
572     ".p2align              4            \n\t"\
573     "1:                                 \n\t"\
574     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
575     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
576     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
577     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
578     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
579     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
580     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
581     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
582     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
583     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
584     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
585     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
586     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
587     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
588     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
589     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
590     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
591     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
592     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
593     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
594     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
595     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
596     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
597     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
598     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
599     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
600     "paddw             %%mm3, %%mm4     \n\t"\
601     "movq              %%mm2, %%mm0     \n\t"\
602     "movq              %%mm5, %%mm6     \n\t"\
603     "movq              %%mm4, %%mm3     \n\t"\
604     "punpcklwd         %%mm2, %%mm2     \n\t"\
605     "punpcklwd         %%mm5, %%mm5     \n\t"\
606     "punpcklwd         %%mm4, %%mm4     \n\t"\
607     "paddw             %%mm1, %%mm2     \n\t"\
608     "paddw             %%mm1, %%mm5     \n\t"\
609     "paddw             %%mm1, %%mm4     \n\t"\
610     "punpckhwd         %%mm0, %%mm0     \n\t"\
611     "punpckhwd         %%mm6, %%mm6     \n\t"\
612     "punpckhwd         %%mm3, %%mm3     \n\t"\
613     "paddw             %%mm7, %%mm0     \n\t"\
614     "paddw             %%mm7, %%mm6     \n\t"\
615     "paddw             %%mm7, %%mm3     \n\t"\
616     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
617     "packuswb          %%mm0, %%mm2     \n\t"\
618     "packuswb          %%mm6, %%mm5     \n\t"\
619     "packuswb          %%mm3, %%mm4     \n\t"\
620
621 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
622
623 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
624     "movq  (%1, "#index", 2), %%mm7     \n\t" /* abuf0[index  ]     */\
625     "movq 8(%1, "#index", 2), %%mm1     \n\t" /* abuf0[index+4]     */\
626     "psraw                $7, %%mm7     \n\t" /* abuf0[index  ] >>7 */\
627     "psraw                $7, %%mm1     \n\t" /* abuf0[index+4] >>7 */\
628     "packuswb          %%mm1, %%mm7     \n\t"
629 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
630
631 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
632     "movq       "#b", "#q2"     \n\t" /* B */\
633     "movq       "#r", "#t"      \n\t" /* R */\
634     "punpcklbw  "#g", "#b"      \n\t" /* GBGBGBGB 0 */\
635     "punpcklbw  "#a", "#r"      \n\t" /* ARARARAR 0 */\
636     "punpckhbw  "#g", "#q2"     \n\t" /* GBGBGBGB 2 */\
637     "punpckhbw  "#a", "#t"      \n\t" /* ARARARAR 2 */\
638     "movq       "#b", "#q0"     \n\t" /* GBGBGBGB 0 */\
639     "movq      "#q2", "#q3"     \n\t" /* GBGBGBGB 2 */\
640     "punpcklwd  "#r", "#q0"     \n\t" /* ARGBARGB 0 */\
641     "punpckhwd  "#r", "#b"      \n\t" /* ARGBARGB 1 */\
642     "punpcklwd  "#t", "#q2"     \n\t" /* ARGBARGB 2 */\
643     "punpckhwd  "#t", "#q3"     \n\t" /* ARGBARGB 3 */\
644 \
645     MOVNTQ(   q0,   (dst, index, 4))\
646     MOVNTQ(    b,  8(dst, index, 4))\
647     MOVNTQ(   q2, 16(dst, index, 4))\
648     MOVNTQ(   q3, 24(dst, index, 4))\
649 \
650     "add      $8, "#index"      \n\t"\
651     "cmp "#dstw", "#index"      \n\t"\
652     " jb      1b                \n\t"
653 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)  REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
654
655 #define REAL_WRITERGB16(dst, dstw, index) \
656     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
657     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
658     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
659     "psrlq           $3, %%mm2  \n\t"\
660 \
661     "movq         %%mm2, %%mm1  \n\t"\
662     "movq         %%mm4, %%mm3  \n\t"\
663 \
664     "punpcklbw    %%mm7, %%mm3  \n\t"\
665     "punpcklbw    %%mm5, %%mm2  \n\t"\
666     "punpckhbw    %%mm7, %%mm4  \n\t"\
667     "punpckhbw    %%mm5, %%mm1  \n\t"\
668 \
669     "psllq           $3, %%mm3  \n\t"\
670     "psllq           $3, %%mm4  \n\t"\
671 \
672     "por          %%mm3, %%mm2  \n\t"\
673     "por          %%mm4, %%mm1  \n\t"\
674 \
675     MOVNTQ(%%mm2,  (dst, index, 2))\
676     MOVNTQ(%%mm1, 8(dst, index, 2))\
677 \
678     "add             $8, "#index"   \n\t"\
679     "cmp        "#dstw", "#index"   \n\t"\
680     " jb             1b             \n\t"
681 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
682
683 #define REAL_WRITERGB15(dst, dstw, index) \
684     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
685     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
686     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
687     "psrlq           $3, %%mm2  \n\t"\
688     "psrlq           $1, %%mm5  \n\t"\
689 \
690     "movq         %%mm2, %%mm1  \n\t"\
691     "movq         %%mm4, %%mm3  \n\t"\
692 \
693     "punpcklbw    %%mm7, %%mm3  \n\t"\
694     "punpcklbw    %%mm5, %%mm2  \n\t"\
695     "punpckhbw    %%mm7, %%mm4  \n\t"\
696     "punpckhbw    %%mm5, %%mm1  \n\t"\
697 \
698     "psllq           $2, %%mm3  \n\t"\
699     "psllq           $2, %%mm4  \n\t"\
700 \
701     "por          %%mm3, %%mm2  \n\t"\
702     "por          %%mm4, %%mm1  \n\t"\
703 \
704     MOVNTQ(%%mm2,  (dst, index, 2))\
705     MOVNTQ(%%mm1, 8(dst, index, 2))\
706 \
707     "add             $8, "#index"   \n\t"\
708     "cmp        "#dstw", "#index"   \n\t"\
709     " jb             1b             \n\t"
710 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
711
712 #define WRITEBGR24OLD(dst, dstw, index) \
713     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
714     "movq      %%mm2, %%mm1             \n\t" /* B */\
715     "movq      %%mm5, %%mm6             \n\t" /* R */\
716     "punpcklbw %%mm4, %%mm2             \n\t" /* GBGBGBGB 0 */\
717     "punpcklbw %%mm7, %%mm5             \n\t" /* 0R0R0R0R 0 */\
718     "punpckhbw %%mm4, %%mm1             \n\t" /* GBGBGBGB 2 */\
719     "punpckhbw %%mm7, %%mm6             \n\t" /* 0R0R0R0R 2 */\
720     "movq      %%mm2, %%mm0             \n\t" /* GBGBGBGB 0 */\
721     "movq      %%mm1, %%mm3             \n\t" /* GBGBGBGB 2 */\
722     "punpcklwd %%mm5, %%mm0             \n\t" /* 0RGB0RGB 0 */\
723     "punpckhwd %%mm5, %%mm2             \n\t" /* 0RGB0RGB 1 */\
724     "punpcklwd %%mm6, %%mm1             \n\t" /* 0RGB0RGB 2 */\
725     "punpckhwd %%mm6, %%mm3             \n\t" /* 0RGB0RGB 3 */\
726 \
727     "movq      %%mm0, %%mm4             \n\t" /* 0RGB0RGB 0 */\
728     "psrlq        $8, %%mm0             \n\t" /* 00RGB0RG 0 */\
729     "pand "MANGLE(bm00000111)", %%mm4   \n\t" /* 00000RGB 0 */\
730     "pand "MANGLE(bm11111000)", %%mm0   \n\t" /* 00RGB000 0.5 */\
731     "por       %%mm4, %%mm0             \n\t" /* 00RGBRGB 0 */\
732     "movq      %%mm2, %%mm4             \n\t" /* 0RGB0RGB 1 */\
733     "psllq       $48, %%mm2             \n\t" /* GB000000 1 */\
734     "por       %%mm2, %%mm0             \n\t" /* GBRGBRGB 0 */\
735 \
736     "movq      %%mm4, %%mm2             \n\t" /* 0RGB0RGB 1 */\
737     "psrld       $16, %%mm4             \n\t" /* 000R000R 1 */\
738     "psrlq       $24, %%mm2             \n\t" /* 0000RGB0 1.5 */\
739     "por       %%mm4, %%mm2             \n\t" /* 000RRGBR 1 */\
740     "pand "MANGLE(bm00001111)", %%mm2   \n\t" /* 0000RGBR 1 */\
741     "movq      %%mm1, %%mm4             \n\t" /* 0RGB0RGB 2 */\
742     "psrlq        $8, %%mm1             \n\t" /* 00RGB0RG 2 */\
743     "pand "MANGLE(bm00000111)", %%mm4   \n\t" /* 00000RGB 2 */\
744     "pand "MANGLE(bm11111000)", %%mm1   \n\t" /* 00RGB000 2.5 */\
745     "por       %%mm4, %%mm1             \n\t" /* 00RGBRGB 2 */\
746     "movq      %%mm1, %%mm4             \n\t" /* 00RGBRGB 2 */\
747     "psllq       $32, %%mm1             \n\t" /* BRGB0000 2 */\
748     "por       %%mm1, %%mm2             \n\t" /* BRGBRGBR 1 */\
749 \
750     "psrlq       $32, %%mm4             \n\t" /* 000000RG 2.5 */\
751     "movq      %%mm3, %%mm5             \n\t" /* 0RGB0RGB 3 */\
752     "psrlq        $8, %%mm3             \n\t" /* 00RGB0RG 3 */\
753     "pand "MANGLE(bm00000111)", %%mm5   \n\t" /* 00000RGB 3 */\
754     "pand "MANGLE(bm11111000)", %%mm3   \n\t" /* 00RGB000 3.5 */\
755     "por       %%mm5, %%mm3             \n\t" /* 00RGBRGB 3 */\
756     "psllq       $16, %%mm3             \n\t" /* RGBRGB00 3 */\
757     "por       %%mm4, %%mm3             \n\t" /* RGBRGBRG 2.5 */\
758 \
759     MOVNTQ(%%mm0,   (dst))\
760     MOVNTQ(%%mm2,  8(dst))\
761     MOVNTQ(%%mm3, 16(dst))\
762     "add         $24, "#dst"            \n\t"\
763 \
764     "add          $8, "#index"          \n\t"\
765     "cmp     "#dstw", "#index"          \n\t"\
766     " jb          1b                    \n\t"
767
768 #define WRITEBGR24MMX(dst, dstw, index) \
769     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
770     "movq      %%mm2, %%mm1     \n\t" /* B */\
771     "movq      %%mm5, %%mm6     \n\t" /* R */\
772     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
773     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
774     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
775     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
776     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
777     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
778     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
779     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
780     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
781     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
782 \
783     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
784     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
785     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
786     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
787 \
788     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
789     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
790     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
791     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
792 \
793     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
794     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
795     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
796     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
797 \
798     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
799     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
800     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
801     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
802     MOVNTQ(%%mm0, (dst))\
803 \
804     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
805     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
806     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
807     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
808     MOVNTQ(%%mm6, 8(dst))\
809 \
810     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
811     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
812     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
813     MOVNTQ(%%mm5, 16(dst))\
814 \
815     "add         $24, "#dst"    \n\t"\
816 \
817     "add          $8, "#index"  \n\t"\
818     "cmp     "#dstw", "#index"  \n\t"\
819     " jb          1b            \n\t"
820
821 #define WRITEBGR24MMX2(dst, dstw, index) \
822     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
823     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
824     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
825     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
826     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
827     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
828 \
829     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
830     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
831     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
832 \
833     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
834     "por    %%mm1, %%mm6        \n\t"\
835     "por    %%mm3, %%mm6        \n\t"\
836     MOVNTQ(%%mm6, (dst))\
837 \
838     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
839     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
840     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
841     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
842 \
843     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
844     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
845     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
846 \
847     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
848     "por    %%mm3, %%mm6        \n\t"\
849     MOVNTQ(%%mm6, 8(dst))\
850 \
851     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
852     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
853     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
854 \
855     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
856     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
857     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
858 \
859     "por    %%mm1, %%mm3        \n\t"\
860     "por    %%mm3, %%mm6        \n\t"\
861     MOVNTQ(%%mm6, 16(dst))\
862 \
863     "add      $24, "#dst"       \n\t"\
864 \
865     "add       $8, "#index"     \n\t"\
866     "cmp  "#dstw", "#index"     \n\t"\
867     " jb       1b               \n\t"
868
869 #if COMPILE_TEMPLATE_MMX2
870 #undef WRITEBGR24
871 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
872 #else
873 #undef WRITEBGR24
874 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
875 #endif
876
877 #define REAL_WRITEYUY2(dst, dstw, index) \
878     "packuswb  %%mm3, %%mm3     \n\t"\
879     "packuswb  %%mm4, %%mm4     \n\t"\
880     "packuswb  %%mm7, %%mm1     \n\t"\
881     "punpcklbw %%mm4, %%mm3     \n\t"\
882     "movq      %%mm1, %%mm7     \n\t"\
883     "punpcklbw %%mm3, %%mm1     \n\t"\
884     "punpckhbw %%mm3, %%mm7     \n\t"\
885 \
886     MOVNTQ(%%mm1, (dst, index, 2))\
887     MOVNTQ(%%mm7, 8(dst, index, 2))\
888 \
889     "add          $8, "#index"  \n\t"\
890     "cmp     "#dstw", "#index"  \n\t"\
891     " jb          1b            \n\t"
892 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
893
894
895 static inline void RENAME(yuv2yuvX)(SwsContext *c, const int16_t *lumFilter, const int16_t **lumSrc, int lumFilterSize,
896                                     const int16_t *chrFilter, const int16_t **chrSrc, int chrFilterSize, const int16_t **alpSrc,
897                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest, uint8_t *aDest, long dstW, long chrDstW)
898 {
899     if(!(c->flags & SWS_BITEXACT)) {
900         if (c->flags & SWS_ACCURATE_RND) {
901             if (uDest) {
902                 YSCALEYUV2YV12X_ACCURATE(   "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
903                 YSCALEYUV2YV12X_ACCURATE(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
904             }
905             if (CONFIG_SWSCALE_ALPHA && aDest) {
906                 YSCALEYUV2YV12X_ACCURATE(   "0", ALP_MMX_FILTER_OFFSET, aDest, dstW)
907             }
908
909             YSCALEYUV2YV12X_ACCURATE("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
910         } else {
911             if (uDest) {
912                 YSCALEYUV2YV12X(   "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
913                 YSCALEYUV2YV12X(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
914             }
915             if (CONFIG_SWSCALE_ALPHA && aDest) {
916                 YSCALEYUV2YV12X(   "0", ALP_MMX_FILTER_OFFSET, aDest, dstW)
917             }
918
919             YSCALEYUV2YV12X("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
920         }
921         return;
922     }
923     yuv2yuvXinC(lumFilter, lumSrc, lumFilterSize,
924                 chrFilter, chrSrc, chrFilterSize,
925                 alpSrc, dest, uDest, vDest, aDest, dstW, chrDstW);
926 }
927
928 static inline void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc, const int16_t *chrSrc, const int16_t *alpSrc,
929                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest, uint8_t *aDest, long dstW, long chrDstW)
930 {
931     if(!(c->flags & SWS_BITEXACT)) {
932         long p= 4;
933         const uint8_t *src[4]= {alpSrc + dstW, lumSrc + dstW, chrSrc + chrDstW, chrSrc + VOFW + chrDstW};
934         uint8_t *dst[4]= {aDest, dest, uDest, vDest};
935         x86_reg counter[4]= {dstW, dstW, chrDstW, chrDstW};
936
937         if (c->flags & SWS_ACCURATE_RND) {
938             while(p--) {
939                 if (dst[p]) {
940                     __asm__ volatile(
941                         YSCALEYUV2YV121_ACCURATE
942                         :: "r" (src[p]), "r" (dst[p] + counter[p]),
943                         "g" (-counter[p])
944                         : "%"REG_a
945                     );
946                 }
947             }
948         } else {
949             while(p--) {
950                 if (dst[p]) {
951                     __asm__ volatile(
952                         YSCALEYUV2YV121
953                         :: "r" (src[p]), "r" (dst[p] + counter[p]),
954                         "g" (-counter[p])
955                         : "%"REG_a
956                     );
957                 }
958             }
959         }
960         return;
961     }
962     yuv2yuv1_c(c, lumSrc, chrSrc, alpSrc, dest, uDest, vDest, aDest,
963                dstW, chrDstW);
964 }
965
966
967 /**
968  * vertical scale YV12 to RGB
969  */
970 static inline void RENAME(yuv2packedX)(SwsContext *c, const int16_t *lumFilter, const int16_t **lumSrc, int lumFilterSize,
971                                        const int16_t *chrFilter, const int16_t **chrSrc, int chrFilterSize,
972                                        const int16_t **alpSrc, uint8_t *dest, long dstW, long dstY)
973 {
974     x86_reg dummy=0;
975     x86_reg dstW_reg = dstW;
976     if(!(c->flags & SWS_BITEXACT)) {
977         if (c->flags & SWS_ACCURATE_RND) {
978             switch(c->dstFormat) {
979             case PIX_FMT_RGB32:
980                 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
981                     YSCALEYUV2PACKEDX_ACCURATE
982                     YSCALEYUV2RGBX
983                     "movq                      %%mm2, "U_TEMP"(%0)  \n\t"
984                     "movq                      %%mm4, "V_TEMP"(%0)  \n\t"
985                     "movq                      %%mm5, "Y_TEMP"(%0)  \n\t"
986                     YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
987                     "movq               "Y_TEMP"(%0), %%mm5         \n\t"
988                     "psraw                        $3, %%mm1         \n\t"
989                     "psraw                        $3, %%mm7         \n\t"
990                     "packuswb                  %%mm7, %%mm1         \n\t"
991                     WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
992
993                     YSCALEYUV2PACKEDX_END
994                 } else {
995                     YSCALEYUV2PACKEDX_ACCURATE
996                     YSCALEYUV2RGBX
997                     "pcmpeqd %%mm7, %%mm7 \n\t"
998                     WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
999
1000                     YSCALEYUV2PACKEDX_END
1001                 }
1002                 return;
1003             case PIX_FMT_BGR24:
1004                 YSCALEYUV2PACKEDX_ACCURATE
1005                 YSCALEYUV2RGBX
1006                 "pxor %%mm7, %%mm7 \n\t"
1007                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
1008                 "add %4, %%"REG_c"                        \n\t"
1009                 WRITEBGR24(%%REGc, %5, %%REGa)
1010
1011
1012                 :: "r" (&c->redDither),
1013                 "m" (dummy), "m" (dummy), "m" (dummy),
1014                 "r" (dest), "m" (dstW_reg)
1015                 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
1016                 );
1017                 return;
1018             case PIX_FMT_RGB555:
1019                 YSCALEYUV2PACKEDX_ACCURATE
1020                 YSCALEYUV2RGBX
1021                 "pxor %%mm7, %%mm7 \n\t"
1022                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1023 #ifdef DITHER1XBPP
1024                 "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
1025                 "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
1026                 "paddusb "RED_DITHER"(%0), %%mm5\n\t"
1027 #endif
1028
1029                 WRITERGB15(%4, %5, %%REGa)
1030                 YSCALEYUV2PACKEDX_END
1031                 return;
1032             case PIX_FMT_RGB565:
1033                 YSCALEYUV2PACKEDX_ACCURATE
1034                 YSCALEYUV2RGBX
1035                 "pxor %%mm7, %%mm7 \n\t"
1036                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1037 #ifdef DITHER1XBPP
1038                 "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
1039                 "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
1040                 "paddusb "RED_DITHER"(%0), %%mm5\n\t"
1041 #endif
1042
1043                 WRITERGB16(%4, %5, %%REGa)
1044                 YSCALEYUV2PACKEDX_END
1045                 return;
1046             case PIX_FMT_YUYV422:
1047                 YSCALEYUV2PACKEDX_ACCURATE
1048                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1049
1050                 "psraw $3, %%mm3    \n\t"
1051                 "psraw $3, %%mm4    \n\t"
1052                 "psraw $3, %%mm1    \n\t"
1053                 "psraw $3, %%mm7    \n\t"
1054                 WRITEYUY2(%4, %5, %%REGa)
1055                 YSCALEYUV2PACKEDX_END
1056                 return;
1057             }
1058         } else {
1059             switch(c->dstFormat) {
1060             case PIX_FMT_RGB32:
1061                 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1062                     YSCALEYUV2PACKEDX
1063                     YSCALEYUV2RGBX
1064                     YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
1065                     "psraw                        $3, %%mm1         \n\t"
1066                     "psraw                        $3, %%mm7         \n\t"
1067                     "packuswb                  %%mm7, %%mm1         \n\t"
1068                     WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1069                     YSCALEYUV2PACKEDX_END
1070                 } else {
1071                     YSCALEYUV2PACKEDX
1072                     YSCALEYUV2RGBX
1073                     "pcmpeqd %%mm7, %%mm7 \n\t"
1074                     WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1075                     YSCALEYUV2PACKEDX_END
1076                 }
1077                 return;
1078             case PIX_FMT_BGR24:
1079                 YSCALEYUV2PACKEDX
1080                 YSCALEYUV2RGBX
1081                 "pxor                    %%mm7, %%mm7       \n\t"
1082                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
1083                 "add                        %4, %%"REG_c"   \n\t"
1084                 WRITEBGR24(%%REGc, %5, %%REGa)
1085
1086                 :: "r" (&c->redDither),
1087                 "m" (dummy), "m" (dummy), "m" (dummy),
1088                 "r" (dest),  "m" (dstW_reg)
1089                 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
1090                 );
1091                 return;
1092             case PIX_FMT_RGB555:
1093                 YSCALEYUV2PACKEDX
1094                 YSCALEYUV2RGBX
1095                 "pxor %%mm7, %%mm7 \n\t"
1096                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1097 #ifdef DITHER1XBPP
1098                 "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
1099                 "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
1100                 "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
1101 #endif
1102
1103                 WRITERGB15(%4, %5, %%REGa)
1104                 YSCALEYUV2PACKEDX_END
1105                 return;
1106             case PIX_FMT_RGB565:
1107                 YSCALEYUV2PACKEDX
1108                 YSCALEYUV2RGBX
1109                 "pxor %%mm7, %%mm7 \n\t"
1110                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1111 #ifdef DITHER1XBPP
1112                 "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
1113                 "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
1114                 "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
1115 #endif
1116
1117                 WRITERGB16(%4, %5, %%REGa)
1118                 YSCALEYUV2PACKEDX_END
1119                 return;
1120             case PIX_FMT_YUYV422:
1121                 YSCALEYUV2PACKEDX
1122                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1123
1124                 "psraw $3, %%mm3    \n\t"
1125                 "psraw $3, %%mm4    \n\t"
1126                 "psraw $3, %%mm1    \n\t"
1127                 "psraw $3, %%mm7    \n\t"
1128                 WRITEYUY2(%4, %5, %%REGa)
1129                 YSCALEYUV2PACKEDX_END
1130                 return;
1131             }
1132         }
1133     }
1134     yuv2packedXinC(c, lumFilter, lumSrc, lumFilterSize,
1135                    chrFilter, chrSrc, chrFilterSize,
1136                    alpSrc, dest, dstW, dstY);
1137 }
1138
1139 /**
1140  * vertical bilinear scale YV12 to RGB
1141  */
1142 static inline void RENAME(yuv2packed2)(SwsContext *c, const uint16_t *buf0, const uint16_t *buf1, const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1143                           const uint16_t *abuf0, const uint16_t *abuf1, uint8_t *dest, int dstW, int yalpha, int uvalpha, int y)
1144 {
1145     if(!(c->flags & SWS_BITEXACT)) {
1146         switch(c->dstFormat) {
1147         //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1148         case PIX_FMT_RGB32:
1149             if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1150 #if ARCH_X86_64
1151                 __asm__ volatile(
1152                     YSCALEYUV2RGB(%%r8, %5)
1153                     YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
1154                     "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1155                     "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1156                     "packuswb            %%mm7, %%mm1       \n\t"
1157                     WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1158
1159                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "r" (dest),
1160                     "a" (&c->redDither)
1161                     ,"r" (abuf0), "r" (abuf1)
1162                     : "%r8"
1163                 );
1164 #else
1165                 *(const uint16_t **)(&c->u_temp)=abuf0;
1166                 *(const uint16_t **)(&c->v_temp)=abuf1;
1167                 __asm__ volatile(
1168                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1169                     "mov        %4, %%"REG_b"               \n\t"
1170                     "push %%"REG_BP"                        \n\t"
1171                     YSCALEYUV2RGB(%%REGBP, %5)
1172                     "push                   %0              \n\t"
1173                     "push                   %1              \n\t"
1174                     "mov          "U_TEMP"(%5), %0          \n\t"
1175                     "mov          "V_TEMP"(%5), %1          \n\t"
1176                     YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
1177                     "psraw                  $3, %%mm1       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1178                     "psraw                  $3, %%mm7       \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1179                     "packuswb            %%mm7, %%mm1       \n\t"
1180                     "pop                    %1              \n\t"
1181                     "pop                    %0              \n\t"
1182                     WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1183                     "pop %%"REG_BP"                         \n\t"
1184                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1185
1186                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1187                     "a" (&c->redDither)
1188                 );
1189 #endif
1190             } else {
1191                 __asm__ volatile(
1192                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1193                     "mov        %4, %%"REG_b"               \n\t"
1194                     "push %%"REG_BP"                        \n\t"
1195                     YSCALEYUV2RGB(%%REGBP, %5)
1196                     "pcmpeqd %%mm7, %%mm7                   \n\t"
1197                     WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1198                     "pop %%"REG_BP"                         \n\t"
1199                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1200
1201                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1202                     "a" (&c->redDither)
1203                 );
1204             }
1205             return;
1206         case PIX_FMT_BGR24:
1207             __asm__ volatile(
1208                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1209                 "mov        %4, %%"REG_b"               \n\t"
1210                 "push %%"REG_BP"                        \n\t"
1211                 YSCALEYUV2RGB(%%REGBP, %5)
1212                 "pxor    %%mm7, %%mm7                   \n\t"
1213                 WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1214                 "pop %%"REG_BP"                         \n\t"
1215                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1216                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1217                 "a" (&c->redDither)
1218             );
1219             return;
1220         case PIX_FMT_RGB555:
1221             __asm__ volatile(
1222                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1223                 "mov        %4, %%"REG_b"               \n\t"
1224                 "push %%"REG_BP"                        \n\t"
1225                 YSCALEYUV2RGB(%%REGBP, %5)
1226                 "pxor    %%mm7, %%mm7                   \n\t"
1227                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1228 #ifdef DITHER1XBPP
1229                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1230                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1231                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1232 #endif
1233
1234                 WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1235                 "pop %%"REG_BP"                         \n\t"
1236                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1237
1238                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1239                 "a" (&c->redDither)
1240             );
1241             return;
1242         case PIX_FMT_RGB565:
1243             __asm__ volatile(
1244                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1245                 "mov        %4, %%"REG_b"               \n\t"
1246                 "push %%"REG_BP"                        \n\t"
1247                 YSCALEYUV2RGB(%%REGBP, %5)
1248                 "pxor    %%mm7, %%mm7                   \n\t"
1249                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1250 #ifdef DITHER1XBPP
1251                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1252                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1253                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1254 #endif
1255
1256                 WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1257                 "pop %%"REG_BP"                         \n\t"
1258                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1259                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1260                 "a" (&c->redDither)
1261             );
1262             return;
1263         case PIX_FMT_YUYV422:
1264             __asm__ volatile(
1265                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1266                 "mov %4, %%"REG_b"                        \n\t"
1267                 "push %%"REG_BP"                        \n\t"
1268                 YSCALEYUV2PACKED(%%REGBP, %5)
1269                 WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1270                 "pop %%"REG_BP"                         \n\t"
1271                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1272                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1273                 "a" (&c->redDither)
1274             );
1275             return;
1276         default: break;
1277         }
1278     }
1279     yuv2packed2_c(c, buf0, buf1, uvbuf0, uvbuf1, abuf0, abuf1,
1280                   dest, dstW, yalpha, uvalpha, y);
1281 }
1282
1283 /**
1284  * YV12 to RGB without scaling or interpolating
1285  */
1286 static inline void RENAME(yuv2packed1)(SwsContext *c, const uint16_t *buf0, const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1287                           const uint16_t *abuf0, uint8_t *dest, int dstW, int uvalpha, enum PixelFormat dstFormat, int flags, int y)
1288 {
1289     if(!(flags & SWS_BITEXACT)) {
1290         const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1291
1292         if (flags&SWS_FULL_CHR_H_INT) {
1293             c->yuv2packed2(c, buf0, buf0, uvbuf0, uvbuf1, abuf0, abuf0, dest, dstW, 0, uvalpha, y);
1294             return;
1295         }
1296
1297         if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1298             switch(dstFormat) {
1299             case PIX_FMT_RGB32:
1300                 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1301                     __asm__ volatile(
1302                         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1303                         "mov        %4, %%"REG_b"               \n\t"
1304                         "push %%"REG_BP"                        \n\t"
1305                         YSCALEYUV2RGB1(%%REGBP, %5)
1306                         YSCALEYUV2RGB1_ALPHA(%%REGBP)
1307                         WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1308                         "pop %%"REG_BP"                         \n\t"
1309                         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1310
1311                         :: "c" (buf0), "d" (abuf0), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1312                         "a" (&c->redDither)
1313                     );
1314                 } else {
1315                     __asm__ volatile(
1316                         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1317                         "mov        %4, %%"REG_b"               \n\t"
1318                         "push %%"REG_BP"                        \n\t"
1319                         YSCALEYUV2RGB1(%%REGBP, %5)
1320                         "pcmpeqd %%mm7, %%mm7                   \n\t"
1321                         WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1322                         "pop %%"REG_BP"                         \n\t"
1323                         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1324
1325                         :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1326                         "a" (&c->redDither)
1327                     );
1328                 }
1329                 return;
1330             case PIX_FMT_BGR24:
1331                 __asm__ volatile(
1332                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1333                     "mov        %4, %%"REG_b"               \n\t"
1334                     "push %%"REG_BP"                        \n\t"
1335                     YSCALEYUV2RGB1(%%REGBP, %5)
1336                     "pxor    %%mm7, %%mm7                   \n\t"
1337                     WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1338                     "pop %%"REG_BP"                         \n\t"
1339                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1340
1341                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1342                     "a" (&c->redDither)
1343                 );
1344                 return;
1345             case PIX_FMT_RGB555:
1346                 __asm__ volatile(
1347                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1348                     "mov        %4, %%"REG_b"               \n\t"
1349                     "push %%"REG_BP"                        \n\t"
1350                     YSCALEYUV2RGB1(%%REGBP, %5)
1351                     "pxor    %%mm7, %%mm7                   \n\t"
1352                     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1353 #ifdef DITHER1XBPP
1354                     "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1355                     "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1356                     "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1357 #endif
1358                     WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1359                     "pop %%"REG_BP"                         \n\t"
1360                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1361
1362                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1363                     "a" (&c->redDither)
1364                 );
1365                 return;
1366             case PIX_FMT_RGB565:
1367                 __asm__ volatile(
1368                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1369                     "mov        %4, %%"REG_b"               \n\t"
1370                     "push %%"REG_BP"                        \n\t"
1371                     YSCALEYUV2RGB1(%%REGBP, %5)
1372                     "pxor    %%mm7, %%mm7                   \n\t"
1373                     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1374 #ifdef DITHER1XBPP
1375                     "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1376                     "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1377                     "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1378 #endif
1379
1380                     WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1381                     "pop %%"REG_BP"                         \n\t"
1382                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1383
1384                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1385                     "a" (&c->redDither)
1386                 );
1387                 return;
1388             case PIX_FMT_YUYV422:
1389                 __asm__ volatile(
1390                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1391                     "mov        %4, %%"REG_b"               \n\t"
1392                     "push %%"REG_BP"                        \n\t"
1393                     YSCALEYUV2PACKED1(%%REGBP, %5)
1394                     WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1395                     "pop %%"REG_BP"                         \n\t"
1396                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1397
1398                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1399                     "a" (&c->redDither)
1400                 );
1401                 return;
1402             }
1403         } else {
1404             switch(dstFormat) {
1405             case PIX_FMT_RGB32:
1406                 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1407                     __asm__ volatile(
1408                         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1409                         "mov        %4, %%"REG_b"               \n\t"
1410                         "push %%"REG_BP"                        \n\t"
1411                         YSCALEYUV2RGB1b(%%REGBP, %5)
1412                         YSCALEYUV2RGB1_ALPHA(%%REGBP)
1413                         WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1414                         "pop %%"REG_BP"                         \n\t"
1415                         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1416
1417                         :: "c" (buf0), "d" (abuf0), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1418                         "a" (&c->redDither)
1419                     );
1420                 } else {
1421                     __asm__ volatile(
1422                         "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1423                         "mov        %4, %%"REG_b"               \n\t"
1424                         "push %%"REG_BP"                        \n\t"
1425                         YSCALEYUV2RGB1b(%%REGBP, %5)
1426                         "pcmpeqd %%mm7, %%mm7                   \n\t"
1427                         WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1428                         "pop %%"REG_BP"                         \n\t"
1429                         "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1430
1431                         :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1432                         "a" (&c->redDither)
1433                     );
1434                 }
1435                 return;
1436             case PIX_FMT_BGR24:
1437                 __asm__ volatile(
1438                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1439                     "mov        %4, %%"REG_b"               \n\t"
1440                     "push %%"REG_BP"                        \n\t"
1441                     YSCALEYUV2RGB1b(%%REGBP, %5)
1442                     "pxor    %%mm7, %%mm7                   \n\t"
1443                     WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1444                     "pop %%"REG_BP"                         \n\t"
1445                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1446
1447                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1448                     "a" (&c->redDither)
1449                 );
1450                 return;
1451             case PIX_FMT_RGB555:
1452                 __asm__ volatile(
1453                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1454                     "mov        %4, %%"REG_b"               \n\t"
1455                     "push %%"REG_BP"                        \n\t"
1456                     YSCALEYUV2RGB1b(%%REGBP, %5)
1457                     "pxor    %%mm7, %%mm7                   \n\t"
1458                     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1459 #ifdef DITHER1XBPP
1460                     "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1461                     "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1462                     "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1463 #endif
1464                     WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1465                     "pop %%"REG_BP"                         \n\t"
1466                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1467
1468                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1469                     "a" (&c->redDither)
1470                 );
1471                 return;
1472             case PIX_FMT_RGB565:
1473                 __asm__ volatile(
1474                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1475                     "mov        %4, %%"REG_b"               \n\t"
1476                     "push %%"REG_BP"                        \n\t"
1477                     YSCALEYUV2RGB1b(%%REGBP, %5)
1478                     "pxor    %%mm7, %%mm7                   \n\t"
1479                     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1480 #ifdef DITHER1XBPP
1481                     "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1482                     "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1483                     "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1484 #endif
1485
1486                     WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1487                     "pop %%"REG_BP"                         \n\t"
1488                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1489
1490                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1491                     "a" (&c->redDither)
1492                 );
1493                 return;
1494             case PIX_FMT_YUYV422:
1495                 __asm__ volatile(
1496                     "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1497                     "mov        %4, %%"REG_b"               \n\t"
1498                     "push %%"REG_BP"                        \n\t"
1499                     YSCALEYUV2PACKED1b(%%REGBP, %5)
1500                     WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1501                     "pop %%"REG_BP"                         \n\t"
1502                     "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1503
1504                     :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1505                     "a" (&c->redDither)
1506                 );
1507                 return;
1508             }
1509         }
1510     }
1511     yuv2packed1_c(c, buf0, uvbuf0, uvbuf1, abuf0, dest,
1512                   dstW, uvalpha, dstFormat, flags, y);
1513 }
1514
1515 //FIXME yuy2* can read up to 7 samples too much
1516
1517 static inline void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1518 {
1519     __asm__ volatile(
1520         "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1521         "mov                    %0, %%"REG_a"       \n\t"
1522         "1:                                         \n\t"
1523         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1524         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1525         "pand                %%mm2, %%mm0           \n\t"
1526         "pand                %%mm2, %%mm1           \n\t"
1527         "packuswb            %%mm1, %%mm0           \n\t"
1528         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1529         "add                    $8, %%"REG_a"       \n\t"
1530         " js                    1b                  \n\t"
1531         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1532         : "%"REG_a
1533     );
1534 }
1535
1536 static inline void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1537 {
1538     __asm__ volatile(
1539         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1540         "mov                    %0, %%"REG_a"       \n\t"
1541         "1:                                         \n\t"
1542         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1543         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1544         "psrlw                  $8, %%mm0           \n\t"
1545         "psrlw                  $8, %%mm1           \n\t"
1546         "packuswb            %%mm1, %%mm0           \n\t"
1547         "movq                %%mm0, %%mm1           \n\t"
1548         "psrlw                  $8, %%mm0           \n\t"
1549         "pand                %%mm4, %%mm1           \n\t"
1550         "packuswb            %%mm0, %%mm0           \n\t"
1551         "packuswb            %%mm1, %%mm1           \n\t"
1552         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1553         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1554         "add                    $4, %%"REG_a"       \n\t"
1555         " js                    1b                  \n\t"
1556         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1557         : "%"REG_a
1558     );
1559     assert(src1 == src2);
1560 }
1561
1562 static inline void RENAME(LEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1563 {
1564     __asm__ volatile(
1565         "mov                    %0, %%"REG_a"       \n\t"
1566         "1:                                         \n\t"
1567         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1568         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1569         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1570         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1571         "psrlw                  $8, %%mm0           \n\t"
1572         "psrlw                  $8, %%mm1           \n\t"
1573         "psrlw                  $8, %%mm2           \n\t"
1574         "psrlw                  $8, %%mm3           \n\t"
1575         "packuswb            %%mm1, %%mm0           \n\t"
1576         "packuswb            %%mm3, %%mm2           \n\t"
1577         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1578         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1579         "add                    $8, %%"REG_a"       \n\t"
1580         " js                    1b                  \n\t"
1581         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1582         : "%"REG_a
1583     );
1584 }
1585
1586 /* This is almost identical to the previous, end exists only because
1587  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1588 static inline void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1589 {
1590     __asm__ volatile(
1591         "mov                  %0, %%"REG_a"         \n\t"
1592         "1:                                         \n\t"
1593         "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1594         "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1595         "psrlw                $8, %%mm0             \n\t"
1596         "psrlw                $8, %%mm1             \n\t"
1597         "packuswb          %%mm1, %%mm0             \n\t"
1598         "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1599         "add                  $8, %%"REG_a"         \n\t"
1600         " js                  1b                    \n\t"
1601         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1602         : "%"REG_a
1603     );
1604 }
1605
1606 static inline void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1607 {
1608     __asm__ volatile(
1609         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1610         "mov                    %0, %%"REG_a"       \n\t"
1611         "1:                                         \n\t"
1612         "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1613         "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1614         "pand                %%mm4, %%mm0           \n\t"
1615         "pand                %%mm4, %%mm1           \n\t"
1616         "packuswb            %%mm1, %%mm0           \n\t"
1617         "movq                %%mm0, %%mm1           \n\t"
1618         "psrlw                  $8, %%mm0           \n\t"
1619         "pand                %%mm4, %%mm1           \n\t"
1620         "packuswb            %%mm0, %%mm0           \n\t"
1621         "packuswb            %%mm1, %%mm1           \n\t"
1622         "movd                %%mm0, (%3, %%"REG_a") \n\t"
1623         "movd                %%mm1, (%2, %%"REG_a") \n\t"
1624         "add                    $4, %%"REG_a"       \n\t"
1625         " js                    1b                  \n\t"
1626         : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1627         : "%"REG_a
1628     );
1629     assert(src1 == src2);
1630 }
1631
1632 static inline void RENAME(BEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1633 {
1634     __asm__ volatile(
1635         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1636         "mov                    %0, %%"REG_a"       \n\t"
1637         "1:                                         \n\t"
1638         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1639         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1640         "movq    (%2, %%"REG_a",2), %%mm2           \n\t"
1641         "movq   8(%2, %%"REG_a",2), %%mm3           \n\t"
1642         "pand                %%mm4, %%mm0           \n\t"
1643         "pand                %%mm4, %%mm1           \n\t"
1644         "pand                %%mm4, %%mm2           \n\t"
1645         "pand                %%mm4, %%mm3           \n\t"
1646         "packuswb            %%mm1, %%mm0           \n\t"
1647         "packuswb            %%mm3, %%mm2           \n\t"
1648         "movq                %%mm0, (%3, %%"REG_a") \n\t"
1649         "movq                %%mm2, (%4, %%"REG_a") \n\t"
1650         "add                    $8, %%"REG_a"       \n\t"
1651         " js                    1b                  \n\t"
1652         : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1653         : "%"REG_a
1654     );
1655 }
1656
1657 static inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1658                                     const uint8_t *src, long width)
1659 {
1660     __asm__ volatile(
1661         "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1662         "mov                    %0, %%"REG_a"       \n\t"
1663         "1:                                         \n\t"
1664         "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1665         "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1666         "movq                %%mm0, %%mm2           \n\t"
1667         "movq                %%mm1, %%mm3           \n\t"
1668         "pand                %%mm4, %%mm0           \n\t"
1669         "pand                %%mm4, %%mm1           \n\t"
1670         "psrlw                  $8, %%mm2           \n\t"
1671         "psrlw                  $8, %%mm3           \n\t"
1672         "packuswb            %%mm1, %%mm0           \n\t"
1673         "packuswb            %%mm3, %%mm2           \n\t"
1674         "movq                %%mm0, (%2, %%"REG_a") \n\t"
1675         "movq                %%mm2, (%3, %%"REG_a") \n\t"
1676         "add                    $8, %%"REG_a"       \n\t"
1677         " js                    1b                  \n\t"
1678         : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1679         : "%"REG_a
1680     );
1681 }
1682
1683 static inline void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1684                                     const uint8_t *src1, const uint8_t *src2,
1685                                     long width, uint32_t *unused)
1686 {
1687     RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1688 }
1689
1690 static inline void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1691                                     const uint8_t *src1, const uint8_t *src2,
1692                                     long width, uint32_t *unused)
1693 {
1694     RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1695 }
1696
1697 static inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, const uint8_t *src, long width, enum PixelFormat srcFormat)
1698 {
1699
1700     if(srcFormat == PIX_FMT_BGR24) {
1701         __asm__ volatile(
1702             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1703             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1704             :
1705         );
1706     } else {
1707         __asm__ volatile(
1708             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1709             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1710             :
1711         );
1712     }
1713
1714     __asm__ volatile(
1715         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1716         "mov                        %2, %%"REG_a"   \n\t"
1717         "pxor                    %%mm7, %%mm7       \n\t"
1718         "1:                                         \n\t"
1719         PREFETCH"               64(%0)              \n\t"
1720         "movd                     (%0), %%mm0       \n\t"
1721         "movd                    2(%0), %%mm1       \n\t"
1722         "movd                    6(%0), %%mm2       \n\t"
1723         "movd                    8(%0), %%mm3       \n\t"
1724         "add                       $12, %0          \n\t"
1725         "punpcklbw               %%mm7, %%mm0       \n\t"
1726         "punpcklbw               %%mm7, %%mm1       \n\t"
1727         "punpcklbw               %%mm7, %%mm2       \n\t"
1728         "punpcklbw               %%mm7, %%mm3       \n\t"
1729         "pmaddwd                 %%mm5, %%mm0       \n\t"
1730         "pmaddwd                 %%mm6, %%mm1       \n\t"
1731         "pmaddwd                 %%mm5, %%mm2       \n\t"
1732         "pmaddwd                 %%mm6, %%mm3       \n\t"
1733         "paddd                   %%mm1, %%mm0       \n\t"
1734         "paddd                   %%mm3, %%mm2       \n\t"
1735         "paddd                   %%mm4, %%mm0       \n\t"
1736         "paddd                   %%mm4, %%mm2       \n\t"
1737         "psrad                     $15, %%mm0       \n\t"
1738         "psrad                     $15, %%mm2       \n\t"
1739         "packssdw                %%mm2, %%mm0       \n\t"
1740         "packuswb                %%mm0, %%mm0       \n\t"
1741         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1742         "add                        $4, %%"REG_a"   \n\t"
1743         " js                        1b              \n\t"
1744     : "+r" (src)
1745     : "r" (dst+width), "g" ((x86_reg)-width)
1746     : "%"REG_a
1747     );
1748 }
1749
1750 static inline void RENAME(bgr24ToUV_mmx)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src, long width, enum PixelFormat srcFormat)
1751 {
1752     __asm__ volatile(
1753         "movq                    24(%4), %%mm6       \n\t"
1754         "mov                        %3, %%"REG_a"   \n\t"
1755         "pxor                    %%mm7, %%mm7       \n\t"
1756         "1:                                         \n\t"
1757         PREFETCH"               64(%0)              \n\t"
1758         "movd                     (%0), %%mm0       \n\t"
1759         "movd                    2(%0), %%mm1       \n\t"
1760         "punpcklbw               %%mm7, %%mm0       \n\t"
1761         "punpcklbw               %%mm7, %%mm1       \n\t"
1762         "movq                    %%mm0, %%mm2       \n\t"
1763         "movq                    %%mm1, %%mm3       \n\t"
1764         "pmaddwd                  (%4), %%mm0       \n\t"
1765         "pmaddwd                 8(%4), %%mm1       \n\t"
1766         "pmaddwd                16(%4), %%mm2       \n\t"
1767         "pmaddwd                 %%mm6, %%mm3       \n\t"
1768         "paddd                   %%mm1, %%mm0       \n\t"
1769         "paddd                   %%mm3, %%mm2       \n\t"
1770
1771         "movd                    6(%0), %%mm1       \n\t"
1772         "movd                    8(%0), %%mm3       \n\t"
1773         "add                       $12, %0          \n\t"
1774         "punpcklbw               %%mm7, %%mm1       \n\t"
1775         "punpcklbw               %%mm7, %%mm3       \n\t"
1776         "movq                    %%mm1, %%mm4       \n\t"
1777         "movq                    %%mm3, %%mm5       \n\t"
1778         "pmaddwd                  (%4), %%mm1       \n\t"
1779         "pmaddwd                 8(%4), %%mm3       \n\t"
1780         "pmaddwd                16(%4), %%mm4       \n\t"
1781         "pmaddwd                 %%mm6, %%mm5       \n\t"
1782         "paddd                   %%mm3, %%mm1       \n\t"
1783         "paddd                   %%mm5, %%mm4       \n\t"
1784
1785         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1786         "paddd                   %%mm3, %%mm0       \n\t"
1787         "paddd                   %%mm3, %%mm2       \n\t"
1788         "paddd                   %%mm3, %%mm1       \n\t"
1789         "paddd                   %%mm3, %%mm4       \n\t"
1790         "psrad                     $15, %%mm0       \n\t"
1791         "psrad                     $15, %%mm2       \n\t"
1792         "psrad                     $15, %%mm1       \n\t"
1793         "psrad                     $15, %%mm4       \n\t"
1794         "packssdw                %%mm1, %%mm0       \n\t"
1795         "packssdw                %%mm4, %%mm2       \n\t"
1796         "packuswb                %%mm0, %%mm0       \n\t"
1797         "packuswb                %%mm2, %%mm2       \n\t"
1798         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1799         "movd                %%mm2, (%2, %%"REG_a") \n\t"
1800         "add                        $4, %%"REG_a"   \n\t"
1801         " js                        1b              \n\t"
1802     : "+r" (src)
1803     : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1804     : "%"REG_a
1805     );
1806 }
1807
1808 static inline void RENAME(bgr24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1809 {
1810     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1811 }
1812
1813 static inline void RENAME(bgr24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1814 {
1815     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1816     assert(src1 == src2);
1817 }
1818
1819 static inline void RENAME(rgb24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1820 {
1821     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1822 }
1823
1824 static inline void RENAME(rgb24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1825 {
1826     assert(src1==src2);
1827     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1828 }
1829
1830
1831 // bilinear / bicubic scaling
1832 static inline void RENAME(hScale)(int16_t *dst, int dstW, const uint8_t *src, int srcW, int xInc,
1833                                   const int16_t *filter, const int16_t *filterPos, long filterSize)
1834 {
1835     assert(filterSize % 4 == 0 && filterSize>0);
1836     if (filterSize==4) { // Always true for upscaling, sometimes for down, too.
1837         x86_reg counter= -2*dstW;
1838         filter-= counter*2;
1839         filterPos-= counter/2;
1840         dst-= counter/2;
1841         __asm__ volatile(
1842 #if defined(PIC)
1843             "push            %%"REG_b"              \n\t"
1844 #endif
1845             "pxor                %%mm7, %%mm7       \n\t"
1846             "push           %%"REG_BP"              \n\t" // we use 7 regs here ...
1847             "mov             %%"REG_a", %%"REG_BP"  \n\t"
1848             ".p2align                4              \n\t"
1849             "1:                                     \n\t"
1850             "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"
1851             "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"
1852             "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"
1853             "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"
1854             "movd      (%3, %%"REG_a"), %%mm0       \n\t"
1855             "movd      (%3, %%"REG_b"), %%mm2       \n\t"
1856             "punpcklbw           %%mm7, %%mm0       \n\t"
1857             "punpcklbw           %%mm7, %%mm2       \n\t"
1858             "pmaddwd             %%mm1, %%mm0       \n\t"
1859             "pmaddwd             %%mm2, %%mm3       \n\t"
1860             "movq                %%mm0, %%mm4       \n\t"
1861             "punpckldq           %%mm3, %%mm0       \n\t"
1862             "punpckhdq           %%mm3, %%mm4       \n\t"
1863             "paddd               %%mm4, %%mm0       \n\t"
1864             "psrad                  $7, %%mm0       \n\t"
1865             "packssdw            %%mm0, %%mm0       \n\t"
1866             "movd                %%mm0, (%4, %%"REG_BP")    \n\t"
1867             "add                    $4, %%"REG_BP"  \n\t"
1868             " jnc                   1b              \n\t"
1869
1870             "pop            %%"REG_BP"              \n\t"
1871 #if defined(PIC)
1872             "pop             %%"REG_b"              \n\t"
1873 #endif
1874             : "+a" (counter)
1875             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1876 #if !defined(PIC)
1877             : "%"REG_b
1878 #endif
1879         );
1880     } else if (filterSize==8) {
1881         x86_reg counter= -2*dstW;
1882         filter-= counter*4;
1883         filterPos-= counter/2;
1884         dst-= counter/2;
1885         __asm__ volatile(
1886 #if defined(PIC)
1887             "push             %%"REG_b"             \n\t"
1888 #endif
1889             "pxor                 %%mm7, %%mm7      \n\t"
1890             "push            %%"REG_BP"             \n\t" // we use 7 regs here ...
1891             "mov              %%"REG_a", %%"REG_BP" \n\t"
1892             ".p2align                 4             \n\t"
1893             "1:                                     \n\t"
1894             "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"
1895             "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"
1896             "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"
1897             "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"
1898             "movd       (%3, %%"REG_a"), %%mm0      \n\t"
1899             "movd       (%3, %%"REG_b"), %%mm2      \n\t"
1900             "punpcklbw            %%mm7, %%mm0      \n\t"
1901             "punpcklbw            %%mm7, %%mm2      \n\t"
1902             "pmaddwd              %%mm1, %%mm0      \n\t"
1903             "pmaddwd              %%mm2, %%mm3      \n\t"
1904
1905             "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"
1906             "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"
1907             "movd      4(%3, %%"REG_a"), %%mm4      \n\t"
1908             "movd      4(%3, %%"REG_b"), %%mm2      \n\t"
1909             "punpcklbw            %%mm7, %%mm4      \n\t"
1910             "punpcklbw            %%mm7, %%mm2      \n\t"
1911             "pmaddwd              %%mm1, %%mm4      \n\t"
1912             "pmaddwd              %%mm2, %%mm5      \n\t"
1913             "paddd                %%mm4, %%mm0      \n\t"
1914             "paddd                %%mm5, %%mm3      \n\t"
1915             "movq                 %%mm0, %%mm4      \n\t"
1916             "punpckldq            %%mm3, %%mm0      \n\t"
1917             "punpckhdq            %%mm3, %%mm4      \n\t"
1918             "paddd                %%mm4, %%mm0      \n\t"
1919             "psrad                   $7, %%mm0      \n\t"
1920             "packssdw             %%mm0, %%mm0      \n\t"
1921             "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"
1922             "add                     $4, %%"REG_BP" \n\t"
1923             " jnc                    1b             \n\t"
1924
1925             "pop             %%"REG_BP"             \n\t"
1926 #if defined(PIC)
1927             "pop              %%"REG_b"             \n\t"
1928 #endif
1929             : "+a" (counter)
1930             : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1931 #if !defined(PIC)
1932             : "%"REG_b
1933 #endif
1934         );
1935     } else {
1936         const uint8_t *offset = src+filterSize;
1937         x86_reg counter= -2*dstW;
1938         //filter-= counter*filterSize/2;
1939         filterPos-= counter/2;
1940         dst-= counter/2;
1941         __asm__ volatile(
1942             "pxor                  %%mm7, %%mm7     \n\t"
1943             ".p2align                  4            \n\t"
1944             "1:                                     \n\t"
1945             "mov                      %2, %%"REG_c" \n\t"
1946             "movzwl      (%%"REG_c", %0), %%eax     \n\t"
1947             "movzwl     2(%%"REG_c", %0), %%edx     \n\t"
1948             "mov                      %5, %%"REG_c" \n\t"
1949             "pxor                  %%mm4, %%mm4     \n\t"
1950             "pxor                  %%mm5, %%mm5     \n\t"
1951             "2:                                     \n\t"
1952             "movq                   (%1), %%mm1     \n\t"
1953             "movq               (%1, %6), %%mm3     \n\t"
1954             "movd (%%"REG_c", %%"REG_a"), %%mm0     \n\t"
1955             "movd (%%"REG_c", %%"REG_d"), %%mm2     \n\t"
1956             "punpcklbw             %%mm7, %%mm0     \n\t"
1957             "punpcklbw             %%mm7, %%mm2     \n\t"
1958             "pmaddwd               %%mm1, %%mm0     \n\t"
1959             "pmaddwd               %%mm2, %%mm3     \n\t"
1960             "paddd                 %%mm3, %%mm5     \n\t"
1961             "paddd                 %%mm0, %%mm4     \n\t"
1962             "add                      $8, %1        \n\t"
1963             "add                      $4, %%"REG_c" \n\t"
1964             "cmp                      %4, %%"REG_c" \n\t"
1965             " jb                      2b            \n\t"
1966             "add                      %6, %1        \n\t"
1967             "movq                  %%mm4, %%mm0     \n\t"
1968             "punpckldq             %%mm5, %%mm4     \n\t"
1969             "punpckhdq             %%mm5, %%mm0     \n\t"
1970             "paddd                 %%mm0, %%mm4     \n\t"
1971             "psrad                    $7, %%mm4     \n\t"
1972             "packssdw              %%mm4, %%mm4     \n\t"
1973             "mov                      %3, %%"REG_a" \n\t"
1974             "movd                  %%mm4, (%%"REG_a", %0)   \n\t"
1975             "add                      $4, %0        \n\t"
1976             " jnc                     1b            \n\t"
1977
1978             : "+r" (counter), "+r" (filter)
1979             : "m" (filterPos), "m" (dst), "m"(offset),
1980             "m" (src), "r" ((x86_reg)filterSize*2)
1981             : "%"REG_a, "%"REG_c, "%"REG_d
1982         );
1983     }
1984 }
1985
1986 #define FAST_BILINEAR_X86 \
1987     "subl    %%edi, %%esi    \n\t" /*  src[xx+1] - src[xx] */                   \
1988     "imull   %%ecx, %%esi    \n\t" /* (src[xx+1] - src[xx])*xalpha */           \
1989     "shll      $16, %%edi    \n\t"                                              \
1990     "addl    %%edi, %%esi    \n\t" /* src[xx+1]*xalpha + src[xx]*(1-xalpha) */  \
1991     "mov        %1, %%"REG_D"\n\t"                                              \
1992     "shrl       $9, %%esi    \n\t"                                              \
1993
1994 static inline void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
1995                                         long dstWidth, const uint8_t *src, int srcW,
1996                                         int xInc)
1997 {
1998 #if COMPILE_TEMPLATE_MMX2
1999     int32_t *filterPos = c->hLumFilterPos;
2000     int16_t *filter    = c->hLumFilter;
2001     int     canMMX2BeUsed  = c->canMMX2BeUsed;
2002     void    *mmx2FilterCode= c->lumMmx2FilterCode;
2003     int i;
2004 #if defined(PIC)
2005     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2006 #endif
2007     if (canMMX2BeUsed) {
2008         __asm__ volatile(
2009 #if defined(PIC)
2010             "mov               %%"REG_b", %5        \n\t"
2011 #endif
2012             "pxor                  %%mm7, %%mm7     \n\t"
2013             "mov                      %0, %%"REG_c" \n\t"
2014             "mov                      %1, %%"REG_D" \n\t"
2015             "mov                      %2, %%"REG_d" \n\t"
2016             "mov                      %3, %%"REG_b" \n\t"
2017             "xor               %%"REG_a", %%"REG_a" \n\t" // i
2018             PREFETCH"        (%%"REG_c")            \n\t"
2019             PREFETCH"      32(%%"REG_c")            \n\t"
2020             PREFETCH"      64(%%"REG_c")            \n\t"
2021
2022 #if ARCH_X86_64
2023
2024 #define CALL_MMX2_FILTER_CODE \
2025             "movl            (%%"REG_b"), %%esi     \n\t"\
2026             "call                    *%4            \n\t"\
2027             "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
2028             "add               %%"REG_S", %%"REG_c" \n\t"\
2029             "add               %%"REG_a", %%"REG_D" \n\t"\
2030             "xor               %%"REG_a", %%"REG_a" \n\t"\
2031
2032 #else
2033
2034 #define CALL_MMX2_FILTER_CODE \
2035             "movl (%%"REG_b"), %%esi        \n\t"\
2036             "call         *%4                       \n\t"\
2037             "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
2038             "add               %%"REG_a", %%"REG_D" \n\t"\
2039             "xor               %%"REG_a", %%"REG_a" \n\t"\
2040
2041 #endif /* ARCH_X86_64 */
2042
2043             CALL_MMX2_FILTER_CODE
2044             CALL_MMX2_FILTER_CODE
2045             CALL_MMX2_FILTER_CODE
2046             CALL_MMX2_FILTER_CODE
2047             CALL_MMX2_FILTER_CODE
2048             CALL_MMX2_FILTER_CODE
2049             CALL_MMX2_FILTER_CODE
2050             CALL_MMX2_FILTER_CODE
2051
2052 #if defined(PIC)
2053             "mov                      %5, %%"REG_b" \n\t"
2054 #endif
2055             :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
2056             "m" (mmx2FilterCode)
2057 #if defined(PIC)
2058             ,"m" (ebxsave)
2059 #endif
2060             : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2061 #if !defined(PIC)
2062             ,"%"REG_b
2063 #endif
2064         );
2065         for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) dst[i] = src[srcW-1]*128;
2066     } else {
2067 #endif /* COMPILE_TEMPLATE_MMX2 */
2068     x86_reg xInc_shr16 = xInc >> 16;
2069     uint16_t xInc_mask = xInc & 0xffff;
2070     x86_reg dstWidth_reg = dstWidth;
2071     //NO MMX just normal asm ...
2072     __asm__ volatile(
2073         "xor %%"REG_a", %%"REG_a"            \n\t" // i
2074         "xor %%"REG_d", %%"REG_d"            \n\t" // xx
2075         "xorl    %%ecx, %%ecx                \n\t" // xalpha
2076         ".p2align                4           \n\t"
2077         "1:                                  \n\t"
2078         "movzbl    (%0, %%"REG_d"), %%edi    \n\t" //src[xx]
2079         "movzbl   1(%0, %%"REG_d"), %%esi    \n\t" //src[xx+1]
2080         FAST_BILINEAR_X86
2081         "movw     %%si, (%%"REG_D", %%"REG_a", 2)   \n\t"
2082         "addw       %4, %%cx                 \n\t" //xalpha += xInc&0xFFFF
2083         "adc        %3, %%"REG_d"            \n\t" //xx+= xInc>>16 + carry
2084
2085         "movzbl    (%0, %%"REG_d"), %%edi    \n\t" //src[xx]
2086         "movzbl   1(%0, %%"REG_d"), %%esi    \n\t" //src[xx+1]
2087         FAST_BILINEAR_X86
2088         "movw     %%si, 2(%%"REG_D", %%"REG_a", 2)  \n\t"
2089         "addw       %4, %%cx                 \n\t" //xalpha += xInc&0xFFFF
2090         "adc        %3, %%"REG_d"            \n\t" //xx+= xInc>>16 + carry
2091
2092
2093         "add        $2, %%"REG_a"            \n\t"
2094         "cmp        %2, %%"REG_a"            \n\t"
2095         " jb        1b                       \n\t"
2096
2097
2098         :: "r" (src), "m" (dst), "m" (dstWidth_reg), "m" (xInc_shr16), "m" (xInc_mask)
2099         : "%"REG_a, "%"REG_d, "%ecx", "%"REG_D, "%esi"
2100     );
2101 #if COMPILE_TEMPLATE_MMX2
2102     } //if MMX2 can't be used
2103 #endif
2104 }
2105
2106 static inline void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst,
2107                                         long dstWidth, const uint8_t *src1,
2108                                         const uint8_t *src2, int srcW, int xInc)
2109 {
2110 #if COMPILE_TEMPLATE_MMX2
2111     int32_t *filterPos = c->hChrFilterPos;
2112     int16_t *filter    = c->hChrFilter;
2113     int     canMMX2BeUsed  = c->canMMX2BeUsed;
2114     void    *mmx2FilterCode= c->chrMmx2FilterCode;
2115     int i;
2116 #if defined(PIC)
2117     DECLARE_ALIGNED(8, uint64_t, ebxsave);
2118 #endif
2119     if (canMMX2BeUsed) {
2120         __asm__ volatile(
2121 #if defined(PIC)
2122             "mov          %%"REG_b", %6         \n\t"
2123 #endif
2124             "pxor             %%mm7, %%mm7      \n\t"
2125             "mov                 %0, %%"REG_c"  \n\t"
2126             "mov                 %1, %%"REG_D"  \n\t"
2127             "mov                 %2, %%"REG_d"  \n\t"
2128             "mov                 %3, %%"REG_b"  \n\t"
2129             "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2130             PREFETCH"   (%%"REG_c")             \n\t"
2131             PREFETCH" 32(%%"REG_c")             \n\t"
2132             PREFETCH" 64(%%"REG_c")             \n\t"
2133
2134             CALL_MMX2_FILTER_CODE
2135             CALL_MMX2_FILTER_CODE
2136             CALL_MMX2_FILTER_CODE
2137             CALL_MMX2_FILTER_CODE
2138             "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2139             "mov                 %5, %%"REG_c"  \n\t" // src
2140             "mov                 %1, %%"REG_D"  \n\t" // buf1
2141             "add              $"AV_STRINGIFY(VOF)", %%"REG_D"  \n\t"
2142             PREFETCH"   (%%"REG_c")             \n\t"
2143             PREFETCH" 32(%%"REG_c")             \n\t"
2144             PREFETCH" 64(%%"REG_c")             \n\t"
2145
2146             CALL_MMX2_FILTER_CODE
2147             CALL_MMX2_FILTER_CODE
2148             CALL_MMX2_FILTER_CODE
2149             CALL_MMX2_FILTER_CODE
2150
2151 #if defined(PIC)
2152             "mov %6, %%"REG_b"    \n\t"
2153 #endif
2154             :: "m" (src1), "m" (dst), "m" (filter), "m" (filterPos),
2155             "m" (mmx2FilterCode), "m" (src2)
2156 #if defined(PIC)
2157             ,"m" (ebxsave)
2158 #endif
2159             : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2160 #if !defined(PIC)
2161             ,"%"REG_b
2162 #endif
2163         );
2164         for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
2165             //printf("%d %d %d\n", dstWidth, i, srcW);
2166             dst[i] = src1[srcW-1]*128;
2167             dst[i+VOFW] = src2[srcW-1]*128;
2168         }
2169     } else {
2170 #endif /* COMPILE_TEMPLATE_MMX2 */
2171         x86_reg xInc_shr16 = (x86_reg) (xInc >> 16);
2172         uint16_t xInc_mask = xInc & 0xffff;
2173         x86_reg dstWidth_reg = dstWidth;
2174         __asm__ volatile(
2175             "xor %%"REG_a", %%"REG_a"               \n\t" // i
2176             "xor %%"REG_d", %%"REG_d"               \n\t" // xx
2177             "xorl    %%ecx, %%ecx                   \n\t" // xalpha
2178             ".p2align    4                          \n\t"
2179             "1:                                     \n\t"
2180             "mov        %0, %%"REG_S"               \n\t"
2181             "movzbl  (%%"REG_S", %%"REG_d"), %%edi  \n\t" //src[xx]
2182             "movzbl 1(%%"REG_S", %%"REG_d"), %%esi  \n\t" //src[xx+1]
2183             FAST_BILINEAR_X86
2184             "movw     %%si, (%%"REG_D", %%"REG_a", 2)   \n\t"
2185
2186             "movzbl    (%5, %%"REG_d"), %%edi       \n\t" //src[xx]
2187             "movzbl   1(%5, %%"REG_d"), %%esi       \n\t" //src[xx+1]
2188             FAST_BILINEAR_X86
2189             "movw     %%si, "AV_STRINGIFY(VOF)"(%%"REG_D", %%"REG_a", 2)   \n\t"
2190
2191             "addw       %4, %%cx                    \n\t" //xalpha += xInc&0xFFFF
2192             "adc        %3, %%"REG_d"               \n\t" //xx+= xInc>>16 + carry
2193             "add        $1, %%"REG_a"               \n\t"
2194             "cmp        %2, %%"REG_a"               \n\t"
2195             " jb        1b                          \n\t"
2196
2197 /* GCC 3.3 makes MPlayer crash on IA-32 machines when using "g" operand here,
2198 which is needed to support GCC 4.0. */
2199 #if ARCH_X86_64 && AV_GCC_VERSION_AT_LEAST(3,4)
2200             :: "m" (src1), "m" (dst), "g" (dstWidth_reg), "m" (xInc_shr16), "m" (xInc_mask),
2201 #else
2202             :: "m" (src1), "m" (dst), "m" (dstWidth_reg), "m" (xInc_shr16), "m" (xInc_mask),
2203 #endif
2204             "r" (src2)
2205             : "%"REG_a, "%"REG_d, "%ecx", "%"REG_D, "%esi"
2206         );
2207 #if COMPILE_TEMPLATE_MMX2
2208     } //if MMX2 can't be used
2209 #endif
2210 }
2211
2212 #define DEBUG_SWSCALE_BUFFERS 0
2213 #define DEBUG_BUFFERS(...) if (DEBUG_SWSCALE_BUFFERS) av_log(c, AV_LOG_DEBUG, __VA_ARGS__)
2214
2215 static int RENAME(swScale)(SwsContext *c, const uint8_t* src[], int srcStride[], int srcSliceY,
2216                            int srcSliceH, uint8_t* dst[], int dstStride[])
2217 {
2218     /* load a few things into local vars to make the code more readable? and faster */
2219     const int srcW= c->srcW;
2220     const int dstW= c->dstW;
2221     const int dstH= c->dstH;
2222     const int chrDstW= c->chrDstW;
2223     const int chrSrcW= c->chrSrcW;
2224     const int lumXInc= c->lumXInc;
2225     const int chrXInc= c->chrXInc;
2226     const enum PixelFormat dstFormat= c->dstFormat;
2227     const int flags= c->flags;
2228     int16_t *vLumFilterPos= c->vLumFilterPos;
2229     int16_t *vChrFilterPos= c->vChrFilterPos;
2230     int16_t *hLumFilterPos= c->hLumFilterPos;
2231     int16_t *hChrFilterPos= c->hChrFilterPos;
2232     int16_t *vLumFilter= c->vLumFilter;
2233     int16_t *vChrFilter= c->vChrFilter;
2234     int16_t *hLumFilter= c->hLumFilter;
2235     int16_t *hChrFilter= c->hChrFilter;
2236     int32_t *lumMmxFilter= c->lumMmxFilter;
2237     int32_t *chrMmxFilter= c->chrMmxFilter;
2238     int32_t av_unused *alpMmxFilter= c->alpMmxFilter;
2239     const int vLumFilterSize= c->vLumFilterSize;
2240     const int vChrFilterSize= c->vChrFilterSize;
2241     const int hLumFilterSize= c->hLumFilterSize;
2242     const int hChrFilterSize= c->hChrFilterSize;
2243     int16_t **lumPixBuf= c->lumPixBuf;
2244     int16_t **chrPixBuf= c->chrPixBuf;
2245     int16_t **alpPixBuf= c->alpPixBuf;
2246     const int vLumBufSize= c->vLumBufSize;
2247     const int vChrBufSize= c->vChrBufSize;
2248     uint8_t *formatConvBuffer= c->formatConvBuffer;
2249     const int chrSrcSliceY= srcSliceY >> c->chrSrcVSubSample;
2250     const int chrSrcSliceH= -((-srcSliceH) >> c->chrSrcVSubSample);
2251     int lastDstY;
2252     uint32_t *pal=c->pal_yuv;
2253
2254     /* vars which will change and which we need to store back in the context */
2255     int dstY= c->dstY;
2256     int lumBufIndex= c->lumBufIndex;
2257     int chrBufIndex= c->chrBufIndex;
2258     int lastInLumBuf= c->lastInLumBuf;
2259     int lastInChrBuf= c->lastInChrBuf;
2260
2261     if (isPacked(c->srcFormat)) {
2262         src[0]=
2263         src[1]=
2264         src[2]=
2265         src[3]= src[0];
2266         srcStride[0]=
2267         srcStride[1]=
2268         srcStride[2]=
2269         srcStride[3]= srcStride[0];
2270     }
2271     srcStride[1]<<= c->vChrDrop;
2272     srcStride[2]<<= c->vChrDrop;
2273
2274     DEBUG_BUFFERS("swScale() %p[%d] %p[%d] %p[%d] %p[%d] -> %p[%d] %p[%d] %p[%d] %p[%d]\n",
2275                   src[0], srcStride[0], src[1], srcStride[1], src[2], srcStride[2], src[3], srcStride[3],
2276                   dst[0], dstStride[0], dst[1], dstStride[1], dst[2], dstStride[2], dst[3], dstStride[3]);
2277     DEBUG_BUFFERS("srcSliceY: %d srcSliceH: %d dstY: %d dstH: %d\n",
2278                    srcSliceY,    srcSliceH,    dstY,    dstH);
2279     DEBUG_BUFFERS("vLumFilterSize: %d vLumBufSize: %d vChrFilterSize: %d vChrBufSize: %d\n",
2280                    vLumFilterSize,    vLumBufSize,    vChrFilterSize,    vChrBufSize);
2281
2282     if (dstStride[0]%8 !=0 || dstStride[1]%8 !=0 || dstStride[2]%8 !=0 || dstStride[3]%8 != 0) {
2283         static int warnedAlready=0; //FIXME move this into the context perhaps
2284         if (flags & SWS_PRINT_INFO && !warnedAlready) {
2285             av_log(c, AV_LOG_WARNING, "Warning: dstStride is not aligned!\n"
2286                    "         ->cannot do aligned memory accesses anymore\n");
2287             warnedAlready=1;
2288         }
2289     }
2290
2291     /* Note the user might start scaling the picture in the middle so this
2292        will not get executed. This is not really intended but works
2293        currently, so people might do it. */
2294     if (srcSliceY ==0) {
2295         lumBufIndex=-1;
2296         chrBufIndex=-1;
2297         dstY=0;
2298         lastInLumBuf= -1;
2299         lastInChrBuf= -1;
2300     }
2301
2302     lastDstY= dstY;
2303
2304     for (;dstY < dstH; dstY++) {
2305         unsigned char *dest =dst[0]+dstStride[0]*dstY;
2306         const int chrDstY= dstY>>c->chrDstVSubSample;
2307         unsigned char *uDest=dst[1]+dstStride[1]*chrDstY;
2308         unsigned char *vDest=dst[2]+dstStride[2]*chrDstY;
2309         unsigned char *aDest=(CONFIG_SWSCALE_ALPHA && alpPixBuf) ? dst[3]+dstStride[3]*dstY : NULL;
2310
2311         const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input
2312         const int firstLumSrcY2= vLumFilterPos[FFMIN(dstY | ((1<<c->chrDstVSubSample) - 1), dstH-1)];
2313         const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input
2314         int lastLumSrcY= firstLumSrcY + vLumFilterSize -1; // Last line needed as input
2315         int lastLumSrcY2=firstLumSrcY2+ vLumFilterSize -1; // Last line needed as input
2316         int lastChrSrcY= firstChrSrcY + vChrFilterSize -1; // Last line needed as input
2317         int enough_lines;
2318
2319         //handle holes (FAST_BILINEAR & weird filters)
2320         if (firstLumSrcY > lastInLumBuf) lastInLumBuf= firstLumSrcY-1;
2321         if (firstChrSrcY > lastInChrBuf) lastInChrBuf= firstChrSrcY-1;
2322         assert(firstLumSrcY >= lastInLumBuf - vLumBufSize + 1);
2323         assert(firstChrSrcY >= lastInChrBuf - vChrBufSize + 1);
2324
2325         DEBUG_BUFFERS("dstY: %d\n", dstY);
2326         DEBUG_BUFFERS("\tfirstLumSrcY: %d lastLumSrcY: %d lastInLumBuf: %d\n",
2327                          firstLumSrcY,    lastLumSrcY,    lastInLumBuf);
2328         DEBUG_BUFFERS("\tfirstChrSrcY: %d lastChrSrcY: %d lastInChrBuf: %d\n",
2329                          firstChrSrcY,    lastChrSrcY,    lastInChrBuf);
2330
2331         // Do we have enough lines in this slice to output the dstY line
2332         enough_lines = lastLumSrcY2 < srcSliceY + srcSliceH && lastChrSrcY < -((-srcSliceY - srcSliceH)>>c->chrSrcVSubSample);
2333
2334         if (!enough_lines) {
2335             lastLumSrcY = srcSliceY + srcSliceH - 1;
2336             lastChrSrcY = chrSrcSliceY + chrSrcSliceH - 1;
2337             DEBUG_BUFFERS("buffering slice: lastLumSrcY %d lastChrSrcY %d\n",
2338                                             lastLumSrcY, lastChrSrcY);
2339         }
2340
2341         //Do horizontal scaling
2342         while(lastInLumBuf < lastLumSrcY) {
2343             const uint8_t *src1= src[0]+(lastInLumBuf + 1 - srcSliceY)*srcStride[0];
2344             const uint8_t *src2= src[3]+(lastInLumBuf + 1 - srcSliceY)*srcStride[3];
2345             lumBufIndex++;
2346             assert(lumBufIndex < 2*vLumBufSize);
2347             assert(lastInLumBuf + 1 - srcSliceY < srcSliceH);
2348             assert(lastInLumBuf + 1 - srcSliceY >= 0);
2349             hyscale_c(c, lumPixBuf[ lumBufIndex ], dstW, src1, srcW, lumXInc,
2350                             hLumFilter, hLumFilterPos, hLumFilterSize,
2351                             formatConvBuffer,
2352                             pal, 0);
2353             if (CONFIG_SWSCALE_ALPHA && alpPixBuf)
2354                 hyscale_c(c, alpPixBuf[ lumBufIndex ], dstW, src2, srcW, lumXInc,
2355                                 hLumFilter, hLumFilterPos, hLumFilterSize,
2356                                 formatConvBuffer,
2357                                 pal, 1);
2358             lastInLumBuf++;
2359             DEBUG_BUFFERS("\t\tlumBufIndex %d: lastInLumBuf: %d\n",
2360                                lumBufIndex,    lastInLumBuf);
2361         }
2362         while(lastInChrBuf < lastChrSrcY) {
2363             const uint8_t *src1= src[1]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[1];
2364             const uint8_t *src2= src[2]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[2];
2365             chrBufIndex++;
2366             assert(chrBufIndex < 2*vChrBufSize);
2367             assert(lastInChrBuf + 1 - chrSrcSliceY < (chrSrcSliceH));
2368             assert(lastInChrBuf + 1 - chrSrcSliceY >= 0);
2369             //FIXME replace parameters through context struct (some at least)
2370
2371             if (c->needs_hcscale)
2372                 hcscale_c(c, chrPixBuf[ chrBufIndex ], chrDstW, src1, src2, chrSrcW, chrXInc,
2373                                 hChrFilter, hChrFilterPos, hChrFilterSize,
2374                                 formatConvBuffer,
2375                                 pal);
2376             lastInChrBuf++;
2377             DEBUG_BUFFERS("\t\tchrBufIndex %d: lastInChrBuf: %d\n",
2378                                chrBufIndex,    lastInChrBuf);
2379         }
2380         //wrap buf index around to stay inside the ring buffer
2381         if (lumBufIndex >= vLumBufSize) lumBufIndex-= vLumBufSize;
2382         if (chrBufIndex >= vChrBufSize) chrBufIndex-= vChrBufSize;
2383         if (!enough_lines)
2384             break; //we can't output a dstY line so let's try with the next slice
2385
2386         c->blueDither= ff_dither8[dstY&1];
2387         if (c->dstFormat == PIX_FMT_RGB555 || c->dstFormat == PIX_FMT_BGR555)
2388             c->greenDither= ff_dither8[dstY&1];
2389         else
2390             c->greenDither= ff_dither4[dstY&1];
2391         c->redDither= ff_dither8[(dstY+1)&1];
2392         if (dstY < dstH-2) {
2393             const int16_t **lumSrcPtr= (const int16_t **) lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2394             const int16_t **chrSrcPtr= (const int16_t **) chrPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2395             const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;
2396             int i;
2397             if (flags & SWS_ACCURATE_RND) {
2398                 int s= APCK_SIZE / 8;
2399                 for (i=0; i<vLumFilterSize; i+=2) {
2400                     *(const void**)&lumMmxFilter[s*i              ]= lumSrcPtr[i  ];
2401                     *(const void**)&lumMmxFilter[s*i+APCK_PTR2/4  ]= lumSrcPtr[i+(vLumFilterSize>1)];
2402                               lumMmxFilter[s*i+APCK_COEF/4  ]=
2403                               lumMmxFilter[s*i+APCK_COEF/4+1]= vLumFilter[dstY*vLumFilterSize + i    ]
2404                         + (vLumFilterSize>1 ? vLumFilter[dstY*vLumFilterSize + i + 1]<<16 : 0);
2405                     if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2406                         *(const void**)&alpMmxFilter[s*i              ]= alpSrcPtr[i  ];
2407                         *(const void**)&alpMmxFilter[s*i+APCK_PTR2/4  ]= alpSrcPtr[i+(vLumFilterSize>1)];
2408                                   alpMmxFilter[s*i+APCK_COEF/4  ]=
2409                                   alpMmxFilter[s*i+APCK_COEF/4+1]= lumMmxFilter[s*i+APCK_COEF/4  ];
2410                     }
2411                 }
2412                 for (i=0; i<vChrFilterSize; i+=2) {
2413                     *(const void**)&chrMmxFilter[s*i              ]= chrSrcPtr[i  ];
2414                     *(const void**)&chrMmxFilter[s*i+APCK_PTR2/4  ]= chrSrcPtr[i+(vChrFilterSize>1)];
2415                               chrMmxFilter[s*i+APCK_COEF/4  ]=
2416                               chrMmxFilter[s*i+APCK_COEF/4+1]= vChrFilter[chrDstY*vChrFilterSize + i    ]
2417                         + (vChrFilterSize>1 ? vChrFilter[chrDstY*vChrFilterSize + i + 1]<<16 : 0);
2418                 }
2419             } else {
2420                 for (i=0; i<vLumFilterSize; i++) {
2421                     lumMmxFilter[4*i+0]= (int32_t)lumSrcPtr[i];
2422                     lumMmxFilter[4*i+1]= (uint64_t)lumSrcPtr[i] >> 32;
2423                     lumMmxFilter[4*i+2]=
2424                     lumMmxFilter[4*i+3]=
2425                         ((uint16_t)vLumFilter[dstY*vLumFilterSize + i])*0x10001;
2426                     if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2427                         alpMmxFilter[4*i+0]= (int32_t)alpSrcPtr[i];
2428                         alpMmxFilter[4*i+1]= (uint64_t)alpSrcPtr[i] >> 32;
2429                         alpMmxFilter[4*i+2]=
2430                         alpMmxFilter[4*i+3]= lumMmxFilter[4*i+2];
2431                     }
2432                 }
2433                 for (i=0; i<vChrFilterSize; i++) {
2434                     chrMmxFilter[4*i+0]= (int32_t)chrSrcPtr[i];
2435                     chrMmxFilter[4*i+1]= (uint64_t)chrSrcPtr[i] >> 32;
2436                     chrMmxFilter[4*i+2]=
2437                     chrMmxFilter[4*i+3]=
2438                         ((uint16_t)vChrFilter[chrDstY*vChrFilterSize + i])*0x10001;
2439                 }
2440             }
2441             if (dstFormat == PIX_FMT_NV12 || dstFormat == PIX_FMT_NV21) {
2442                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2443                 if (dstY&chrSkipMask) uDest= NULL; //FIXME split functions in lumi / chromi
2444                 c->yuv2nv12X(c,
2445                              vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2446                              vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2447                              dest, uDest, dstW, chrDstW, dstFormat);
2448             } else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12 like
2449                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2450                 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
2451                 if (is16BPS(dstFormat) || is9_OR_10BPS(dstFormat)) {
2452                     yuv2yuvX16inC(
2453                                   vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2454                                   vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2455                                   alpSrcPtr, (uint16_t *) dest, (uint16_t *) uDest, (uint16_t *) vDest, (uint16_t *) aDest, dstW, chrDstW,
2456                                   dstFormat);
2457                 } else if (vLumFilterSize == 1 && vChrFilterSize == 1) { // unscaled YV12
2458                     const int16_t *lumBuf = lumSrcPtr[0];
2459                     const int16_t *chrBuf= chrSrcPtr[0];
2460                     const int16_t *alpBuf= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? alpSrcPtr[0] : NULL;
2461                     c->yuv2yuv1(c, lumBuf, chrBuf, alpBuf, dest, uDest, vDest, aDest, dstW, chrDstW);
2462                 } else { //General YV12
2463                     c->yuv2yuvX(c,
2464                                 vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2465                                 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2466                                 alpSrcPtr, dest, uDest, vDest, aDest, dstW, chrDstW);
2467                 }
2468             } else {
2469                 assert(lumSrcPtr + vLumFilterSize - 1 < lumPixBuf + vLumBufSize*2);
2470                 assert(chrSrcPtr + vChrFilterSize - 1 < chrPixBuf + vChrBufSize*2);
2471                 if (vLumFilterSize == 1 && vChrFilterSize == 2) { //unscaled RGB
2472                     int chrAlpha= vChrFilter[2*dstY+1];
2473                     if(flags & SWS_FULL_CHR_H_INT) {
2474                         yuv2rgbXinC_full(c, //FIXME write a packed1_full function
2475                                          vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2476                                          vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2477                                          alpSrcPtr, dest, dstW, dstY);
2478                     } else {
2479                         c->yuv2packed1(c, *lumSrcPtr, *chrSrcPtr, *(chrSrcPtr+1),
2480                                        alpPixBuf ? *alpSrcPtr : NULL,
2481                                        dest, dstW, chrAlpha, dstFormat, flags, dstY);
2482                     }
2483                 } else if (vLumFilterSize == 2 && vChrFilterSize == 2) { //bilinear upscale RGB
2484                     int lumAlpha= vLumFilter[2*dstY+1];
2485                     int chrAlpha= vChrFilter[2*dstY+1];
2486                     lumMmxFilter[2]=
2487                     lumMmxFilter[3]= vLumFilter[2*dstY   ]*0x10001;
2488                     chrMmxFilter[2]=
2489                     chrMmxFilter[3]= vChrFilter[2*chrDstY]*0x10001;
2490                     if(flags & SWS_FULL_CHR_H_INT) {
2491                         yuv2rgbXinC_full(c, //FIXME write a packed2_full function
2492                                          vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2493                                          vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2494                                          alpSrcPtr, dest, dstW, dstY);
2495                     } else {
2496                         c->yuv2packed2(c, *lumSrcPtr, *(lumSrcPtr+1), *chrSrcPtr, *(chrSrcPtr+1),
2497                                        alpPixBuf ? *alpSrcPtr : NULL, alpPixBuf ? *(alpSrcPtr+1) : NULL,
2498                                        dest, dstW, lumAlpha, chrAlpha, dstY);
2499                     }
2500                 } else { //general RGB
2501                     if(flags & SWS_FULL_CHR_H_INT) {
2502                         yuv2rgbXinC_full(c,
2503                                          vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2504                                          vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2505                                          alpSrcPtr, dest, dstW, dstY);
2506                     } else {
2507                         c->yuv2packedX(c,
2508                                        vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2509                                        vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2510                                        alpSrcPtr, dest, dstW, dstY);
2511                     }
2512                 }
2513             }
2514         } else { // hmm looks like we can't use MMX here without overwriting this array's tail
2515             const int16_t **lumSrcPtr= (const int16_t **)lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2516             const int16_t **chrSrcPtr= (const int16_t **)chrPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2517             const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **)alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;
2518             if (dstFormat == PIX_FMT_NV12 || dstFormat == PIX_FMT_NV21) {
2519                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2520                 if (dstY&chrSkipMask) uDest= NULL; //FIXME split functions in lumi / chromi
2521                 yuv2nv12XinC(
2522                              vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2523                              vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2524                              dest, uDest, dstW, chrDstW, dstFormat);
2525             } else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12
2526                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2527                 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
2528                 if (is16BPS(dstFormat) || is9_OR_10BPS(dstFormat)) {
2529                     yuv2yuvX16inC(
2530                                   vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2531                                   vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2532                                   alpSrcPtr, (uint16_t *) dest, (uint16_t *) uDest, (uint16_t *) vDest, (uint16_t *) aDest, dstW, chrDstW,
2533                                   dstFormat);
2534                 } else {
2535                     yuv2yuvXinC(
2536                                 vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2537                                 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2538                                 alpSrcPtr, dest, uDest, vDest, aDest, dstW, chrDstW);
2539                 }
2540             } else {
2541                 assert(lumSrcPtr + vLumFilterSize - 1 < lumPixBuf + vLumBufSize*2);
2542                 assert(chrSrcPtr + vChrFilterSize - 1 < chrPixBuf + vChrBufSize*2);
2543                 if(flags & SWS_FULL_CHR_H_INT) {
2544                     yuv2rgbXinC_full(c,
2545                                      vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2546                                      vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2547                                      alpSrcPtr, dest, dstW, dstY);
2548                 } else {
2549                     yuv2packedXinC(c,
2550                                    vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2551                                    vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2552                                    alpSrcPtr, dest, dstW, dstY);
2553                 }
2554             }
2555         }
2556     }
2557
2558     if ((dstFormat == PIX_FMT_YUVA420P) && !alpPixBuf)
2559         fillPlane(dst[3], dstStride[3], dstW, dstY-lastDstY, lastDstY, 255);
2560
2561     if (COMPILE_TEMPLATE_MMX2)      __asm__ volatile("sfence":::"memory");
2562     /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */
2563     if (COMPILE_TEMPLATE_AMD3DNOW)  __asm__ volatile("femms" :::"memory");
2564     else                            __asm__ volatile("emms"  :::"memory");
2565
2566     /* store changed local vars back in the context */
2567     c->dstY= dstY;
2568     c->lumBufIndex= lumBufIndex;
2569     c->chrBufIndex= chrBufIndex;
2570     c->lastInLumBuf= lastInLumBuf;
2571     c->lastInChrBuf= lastInChrBuf;
2572
2573     return dstY - lastDstY;
2574 }
2575
2576 static void RENAME(sws_init_swScale)(SwsContext *c)
2577 {
2578     enum PixelFormat srcFormat = c->srcFormat;
2579
2580     c->yuv2yuv1     = RENAME(yuv2yuv1    );
2581     c->yuv2yuvX     = RENAME(yuv2yuvX    );
2582     c->yuv2packed1  = RENAME(yuv2packed1 );
2583     c->yuv2packed2  = RENAME(yuv2packed2 );
2584     c->yuv2packedX  = RENAME(yuv2packedX );
2585
2586     c->hScale       = RENAME(hScale      );
2587
2588     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2589     if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
2590     {
2591         c->hyscale_fast = RENAME(hyscale_fast);
2592         c->hcscale_fast = RENAME(hcscale_fast);
2593     } else {
2594         c->hyscale_fast = NULL;
2595         c->hcscale_fast = NULL;
2596     }
2597
2598     switch(srcFormat) {
2599         case PIX_FMT_YUYV422  : c->chrToYV12 = RENAME(yuy2ToUV); break;
2600         case PIX_FMT_UYVY422  : c->chrToYV12 = RENAME(uyvyToUV); break;
2601         case PIX_FMT_NV12     : c->chrToYV12 = RENAME(nv12ToUV); break;
2602         case PIX_FMT_NV21     : c->chrToYV12 = RENAME(nv21ToUV); break;
2603         case PIX_FMT_YUV420P16BE:
2604         case PIX_FMT_YUV422P16BE:
2605         case PIX_FMT_YUV444P16BE: c->chrToYV12 = RENAME(BEToUV); break;
2606         case PIX_FMT_YUV420P16LE:
2607         case PIX_FMT_YUV422P16LE:
2608         case PIX_FMT_YUV444P16LE: c->chrToYV12 = RENAME(LEToUV); break;
2609         default: break;
2610     }
2611     if (!c->chrSrcHSubSample) {
2612         switch(srcFormat) {
2613         case PIX_FMT_BGR24  : c->chrToYV12 = RENAME(bgr24ToUV); break;
2614         case PIX_FMT_RGB24  : c->chrToYV12 = RENAME(rgb24ToUV); break;
2615         default: break;
2616         }
2617     }
2618
2619     switch (srcFormat) {
2620     case PIX_FMT_YUYV422  :
2621     case PIX_FMT_YUV420P16BE:
2622     case PIX_FMT_YUV422P16BE:
2623     case PIX_FMT_YUV444P16BE:
2624     case PIX_FMT_Y400A    :
2625     case PIX_FMT_GRAY16BE : c->lumToYV12 = RENAME(yuy2ToY); break;
2626     case PIX_FMT_UYVY422  :
2627     case PIX_FMT_YUV420P16LE:
2628     case PIX_FMT_YUV422P16LE:
2629     case PIX_FMT_YUV444P16LE:
2630     case PIX_FMT_GRAY16LE : c->lumToYV12 = RENAME(uyvyToY); break;
2631     case PIX_FMT_BGR24    : c->lumToYV12 = RENAME(bgr24ToY); break;
2632     case PIX_FMT_RGB24    : c->lumToYV12 = RENAME(rgb24ToY); break;
2633     default: break;
2634     }
2635     if (c->alpPixBuf) {
2636         switch (srcFormat) {
2637         case PIX_FMT_Y400A  : c->alpToYV12 = RENAME(yuy2ToY); break;
2638         default: break;
2639         }
2640     }
2641 }