5754c10871eca1ebc29b213e22c4016e3e5e3239
[ffmpeg.git] / libswscale / swscale_template.c
1 /*
2  * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License as published by
8  * the Free Software Foundation; either version 2 of the License, or
9  * (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  *
20  * The C code (not assembly, MMX, ...) of this file can be used
21  * under the LGPL license.
22  */
23
24 #undef REAL_MOVNTQ
25 #undef MOVNTQ
26 #undef PAVGB
27 #undef PREFETCH
28 #undef PREFETCHW
29 #undef EMMS
30 #undef SFENCE
31
32 #ifdef HAVE_3DNOW
33 /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */
34 #define EMMS     "femms"
35 #else
36 #define EMMS     "emms"
37 #endif
38
39 #ifdef HAVE_3DNOW
40 #define PREFETCH  "prefetch"
41 #define PREFETCHW "prefetchw"
42 #elif defined (HAVE_MMX2)
43 #define PREFETCH "prefetchnta"
44 #define PREFETCHW "prefetcht0"
45 #else
46 #define PREFETCH  " # nop"
47 #define PREFETCHW " # nop"
48 #endif
49
50 #ifdef HAVE_MMX2
51 #define SFENCE "sfence"
52 #else
53 #define SFENCE " # nop"
54 #endif
55
56 #ifdef HAVE_MMX2
57 #define PAVGB(a,b) "pavgb " #a ", " #b " \n\t"
58 #elif defined (HAVE_3DNOW)
59 #define PAVGB(a,b) "pavgusb " #a ", " #b " \n\t"
60 #endif
61
62 #ifdef HAVE_MMX2
63 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
64 #else
65 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
66 #endif
67 #define MOVNTQ(a,b)  REAL_MOVNTQ(a,b)
68
69 #ifdef HAVE_ALTIVEC
70 #include "swscale_altivec_template.c"
71 #endif
72
73 #define YSCALEYUV2YV12X(x, offset, dest, width) \
74     __asm__ volatile(\
75     "xor                          %%"REG_a", %%"REG_a"  \n\t"\
76     "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
77     "movq                             %%mm3, %%mm4      \n\t"\
78     "lea                     " offset "(%0), %%"REG_d"  \n\t"\
79     "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
80     ASMALIGN(4) /* FIXME Unroll? */\
81     "1:                                                 \n\t"\
82     "movq                      8(%%"REG_d"), %%mm0      \n\t" /* filterCoeff */\
83     "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm2      \n\t" /* srcData */\
84     "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm5      \n\t" /* srcData */\
85     "add                                $16, %%"REG_d"  \n\t"\
86     "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
87     "test                         %%"REG_S", %%"REG_S"  \n\t"\
88     "pmulhw                           %%mm0, %%mm2      \n\t"\
89     "pmulhw                           %%mm0, %%mm5      \n\t"\
90     "paddw                            %%mm2, %%mm3      \n\t"\
91     "paddw                            %%mm5, %%mm4      \n\t"\
92     " jnz                                1b             \n\t"\
93     "psraw                               $3, %%mm3      \n\t"\
94     "psraw                               $3, %%mm4      \n\t"\
95     "packuswb                         %%mm4, %%mm3      \n\t"\
96     MOVNTQ(%%mm3, (%1, %%REGa))\
97     "add                                 $8, %%"REG_a"  \n\t"\
98     "cmp                                 %2, %%"REG_a"  \n\t"\
99     "movq             "VROUNDER_OFFSET"(%0), %%mm3      \n\t"\
100     "movq                             %%mm3, %%mm4      \n\t"\
101     "lea                     " offset "(%0), %%"REG_d"  \n\t"\
102     "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
103     "jb                                  1b             \n\t"\
104     :: "r" (&c->redDither),\
105     "r" (dest), "g" (width)\
106     : "%"REG_a, "%"REG_d, "%"REG_S\
107     );
108
109 #define YSCALEYUV2YV12X_ACCURATE(x, offset, dest, width) \
110     __asm__ volatile(\
111     "lea                     " offset "(%0), %%"REG_d"  \n\t"\
112     "xor                          %%"REG_a", %%"REG_a"  \n\t"\
113     "pxor                             %%mm4, %%mm4      \n\t"\
114     "pxor                             %%mm5, %%mm5      \n\t"\
115     "pxor                             %%mm6, %%mm6      \n\t"\
116     "pxor                             %%mm7, %%mm7      \n\t"\
117     "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
118     ASMALIGN(4) \
119     "1:                                                 \n\t"\
120     "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm0      \n\t" /* srcData */\
121     "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm2      \n\t" /* srcData */\
122     "mov        "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"  \n\t"\
123     "movq   "  x "(%%"REG_S", %%"REG_a", 2), %%mm1      \n\t" /* srcData */\
124     "movq                             %%mm0, %%mm3      \n\t"\
125     "punpcklwd                        %%mm1, %%mm0      \n\t"\
126     "punpckhwd                        %%mm1, %%mm3      \n\t"\
127     "movq       "STR(APCK_COEF)"(%%"REG_d"), %%mm1      \n\t" /* filterCoeff */\
128     "pmaddwd                          %%mm1, %%mm0      \n\t"\
129     "pmaddwd                          %%mm1, %%mm3      \n\t"\
130     "paddd                            %%mm0, %%mm4      \n\t"\
131     "paddd                            %%mm3, %%mm5      \n\t"\
132     "movq 8+"  x "(%%"REG_S", %%"REG_a", 2), %%mm3      \n\t" /* srcData */\
133     "mov        "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"  \n\t"\
134     "add                  $"STR(APCK_SIZE)", %%"REG_d"  \n\t"\
135     "test                         %%"REG_S", %%"REG_S"  \n\t"\
136     "movq                             %%mm2, %%mm0      \n\t"\
137     "punpcklwd                        %%mm3, %%mm2      \n\t"\
138     "punpckhwd                        %%mm3, %%mm0      \n\t"\
139     "pmaddwd                          %%mm1, %%mm2      \n\t"\
140     "pmaddwd                          %%mm1, %%mm0      \n\t"\
141     "paddd                            %%mm2, %%mm6      \n\t"\
142     "paddd                            %%mm0, %%mm7      \n\t"\
143     " jnz                                1b             \n\t"\
144     "psrad                              $16, %%mm4      \n\t"\
145     "psrad                              $16, %%mm5      \n\t"\
146     "psrad                              $16, %%mm6      \n\t"\
147     "psrad                              $16, %%mm7      \n\t"\
148     "movq             "VROUNDER_OFFSET"(%0), %%mm0      \n\t"\
149     "packssdw                         %%mm5, %%mm4      \n\t"\
150     "packssdw                         %%mm7, %%mm6      \n\t"\
151     "paddw                            %%mm0, %%mm4      \n\t"\
152     "paddw                            %%mm0, %%mm6      \n\t"\
153     "psraw                               $3, %%mm4      \n\t"\
154     "psraw                               $3, %%mm6      \n\t"\
155     "packuswb                         %%mm6, %%mm4      \n\t"\
156     MOVNTQ(%%mm4, (%1, %%REGa))\
157     "add                                 $8, %%"REG_a"  \n\t"\
158     "cmp                                 %2, %%"REG_a"  \n\t"\
159     "lea                     " offset "(%0), %%"REG_d"  \n\t"\
160     "pxor                             %%mm4, %%mm4      \n\t"\
161     "pxor                             %%mm5, %%mm5      \n\t"\
162     "pxor                             %%mm6, %%mm6      \n\t"\
163     "pxor                             %%mm7, %%mm7      \n\t"\
164     "mov                        (%%"REG_d"), %%"REG_S"  \n\t"\
165     "jb                                  1b             \n\t"\
166     :: "r" (&c->redDither),\
167     "r" (dest), "g" (width)\
168     : "%"REG_a, "%"REG_d, "%"REG_S\
169     );
170
171 #define YSCALEYUV2YV121 \
172     "mov %2, %%"REG_a"                    \n\t"\
173     ASMALIGN(4) /* FIXME Unroll? */\
174     "1:                                   \n\t"\
175     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
176     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
177     "psraw                 $7, %%mm0      \n\t"\
178     "psraw                 $7, %%mm1      \n\t"\
179     "packuswb           %%mm1, %%mm0      \n\t"\
180     MOVNTQ(%%mm0, (%1, %%REGa))\
181     "add                   $8, %%"REG_a"  \n\t"\
182     "jnc                   1b             \n\t"
183
184 #define YSCALEYUV2YV121_ACCURATE \
185     "mov %2, %%"REG_a"                    \n\t"\
186     "pcmpeqw %%mm7, %%mm7                 \n\t"\
187     "psrlw                 $15, %%mm7     \n\t"\
188     "psllw                  $6, %%mm7     \n\t"\
189     ASMALIGN(4) /* FIXME Unroll? */\
190     "1:                                   \n\t"\
191     "movq  (%0, %%"REG_a", 2), %%mm0      \n\t"\
192     "movq 8(%0, %%"REG_a", 2), %%mm1      \n\t"\
193     "paddsw             %%mm7, %%mm0      \n\t"\
194     "paddsw             %%mm7, %%mm1      \n\t"\
195     "psraw                 $7, %%mm0      \n\t"\
196     "psraw                 $7, %%mm1      \n\t"\
197     "packuswb           %%mm1, %%mm0      \n\t"\
198     MOVNTQ(%%mm0, (%1, %%REGa))\
199     "add                   $8, %%"REG_a"  \n\t"\
200     "jnc                   1b             \n\t"
201
202 /*
203     :: "m" (-lumFilterSize), "m" (-chrFilterSize),
204        "m" (lumMmxFilter+lumFilterSize*4), "m" (chrMmxFilter+chrFilterSize*4),
205        "r" (dest), "m" (dstW),
206        "m" (lumSrc+lumFilterSize), "m" (chrSrc+chrFilterSize)
207     : "%eax", "%ebx", "%ecx", "%edx", "%esi"
208 */
209 #define YSCALEYUV2PACKEDX \
210     __asm__ volatile(\
211     "xor                   %%"REG_a", %%"REG_a"     \n\t"\
212     ASMALIGN(4)\
213     "nop                                            \n\t"\
214     "1:                                             \n\t"\
215     "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
216     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
217     "movq      "VROUNDER_OFFSET"(%0), %%mm3         \n\t"\
218     "movq                      %%mm3, %%mm4         \n\t"\
219     ASMALIGN(4)\
220     "2:                                             \n\t"\
221     "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
222     "movq     (%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* UsrcData */\
223     "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm5         \n\t" /* VsrcData */\
224     "add                         $16, %%"REG_d"     \n\t"\
225     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
226     "pmulhw                    %%mm0, %%mm2         \n\t"\
227     "pmulhw                    %%mm0, %%mm5         \n\t"\
228     "paddw                     %%mm2, %%mm3         \n\t"\
229     "paddw                     %%mm5, %%mm4         \n\t"\
230     "test                  %%"REG_S", %%"REG_S"     \n\t"\
231     " jnz                         2b                \n\t"\
232 \
233     "lea "LUM_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
234     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
235     "movq      "VROUNDER_OFFSET"(%0), %%mm1         \n\t"\
236     "movq                      %%mm1, %%mm7         \n\t"\
237     ASMALIGN(4)\
238     "2:                                             \n\t"\
239     "movq               8(%%"REG_d"), %%mm0         \n\t" /* filterCoeff */\
240     "movq  (%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y1srcData */\
241     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm5         \n\t" /* Y2srcData */\
242     "add                         $16, %%"REG_d"            \n\t"\
243     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
244     "pmulhw                    %%mm0, %%mm2         \n\t"\
245     "pmulhw                    %%mm0, %%mm5         \n\t"\
246     "paddw                     %%mm2, %%mm1         \n\t"\
247     "paddw                     %%mm5, %%mm7         \n\t"\
248     "test                  %%"REG_S", %%"REG_S"     \n\t"\
249     " jnz                         2b                \n\t"\
250
251 #define YSCALEYUV2PACKEDX_END                 \
252     :: "r" (&c->redDither),                   \
253         "m" (dummy), "m" (dummy), "m" (dummy),\
254         "r" (dest), "m" (dstW)                \
255     : "%"REG_a, "%"REG_d, "%"REG_S            \
256     );
257
258 #define YSCALEYUV2PACKEDX_ACCURATE \
259     __asm__ volatile(\
260     "xor %%"REG_a", %%"REG_a"                       \n\t"\
261     ASMALIGN(4)\
262     "nop                                            \n\t"\
263     "1:                                             \n\t"\
264     "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
265     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
266     "pxor                      %%mm4, %%mm4         \n\t"\
267     "pxor                      %%mm5, %%mm5         \n\t"\
268     "pxor                      %%mm6, %%mm6         \n\t"\
269     "pxor                      %%mm7, %%mm7         \n\t"\
270     ASMALIGN(4)\
271     "2:                                             \n\t"\
272     "movq     (%%"REG_S", %%"REG_a"), %%mm0         \n\t" /* UsrcData */\
273     "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm2         \n\t" /* VsrcData */\
274     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
275     "movq     (%%"REG_S", %%"REG_a"), %%mm1         \n\t" /* UsrcData */\
276     "movq                      %%mm0, %%mm3         \n\t"\
277     "punpcklwd                 %%mm1, %%mm0         \n\t"\
278     "punpckhwd                 %%mm1, %%mm3         \n\t"\
279     "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1         \n\t" /* filterCoeff */\
280     "pmaddwd                   %%mm1, %%mm0         \n\t"\
281     "pmaddwd                   %%mm1, %%mm3         \n\t"\
282     "paddd                     %%mm0, %%mm4         \n\t"\
283     "paddd                     %%mm3, %%mm5         \n\t"\
284     "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm3         \n\t" /* VsrcData */\
285     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
286     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
287     "test                  %%"REG_S", %%"REG_S"     \n\t"\
288     "movq                      %%mm2, %%mm0         \n\t"\
289     "punpcklwd                 %%mm3, %%mm2         \n\t"\
290     "punpckhwd                 %%mm3, %%mm0         \n\t"\
291     "pmaddwd                   %%mm1, %%mm2         \n\t"\
292     "pmaddwd                   %%mm1, %%mm0         \n\t"\
293     "paddd                     %%mm2, %%mm6         \n\t"\
294     "paddd                     %%mm0, %%mm7         \n\t"\
295     " jnz                         2b                \n\t"\
296     "psrad                       $16, %%mm4         \n\t"\
297     "psrad                       $16, %%mm5         \n\t"\
298     "psrad                       $16, %%mm6         \n\t"\
299     "psrad                       $16, %%mm7         \n\t"\
300     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
301     "packssdw                  %%mm5, %%mm4         \n\t"\
302     "packssdw                  %%mm7, %%mm6         \n\t"\
303     "paddw                     %%mm0, %%mm4         \n\t"\
304     "paddw                     %%mm0, %%mm6         \n\t"\
305     "movq                      %%mm4, "U_TEMP"(%0)  \n\t"\
306     "movq                      %%mm6, "V_TEMP"(%0)  \n\t"\
307 \
308     "lea "LUM_MMX_FILTER_OFFSET"(%0), %%"REG_d"     \n\t"\
309     "mov                 (%%"REG_d"), %%"REG_S"     \n\t"\
310     "pxor                      %%mm1, %%mm1         \n\t"\
311     "pxor                      %%mm5, %%mm5         \n\t"\
312     "pxor                      %%mm7, %%mm7         \n\t"\
313     "pxor                      %%mm6, %%mm6         \n\t"\
314     ASMALIGN(4)\
315     "2:                                             \n\t"\
316     "movq  (%%"REG_S", %%"REG_a", 2), %%mm0         \n\t" /* Y1srcData */\
317     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2         \n\t" /* Y2srcData */\
318     "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S"     \n\t"\
319     "movq  (%%"REG_S", %%"REG_a", 2), %%mm4         \n\t" /* Y1srcData */\
320     "movq                      %%mm0, %%mm3         \n\t"\
321     "punpcklwd                 %%mm4, %%mm0         \n\t"\
322     "punpckhwd                 %%mm4, %%mm3         \n\t"\
323     "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4         \n\t" /* filterCoeff */\
324     "pmaddwd                   %%mm4, %%mm0         \n\t"\
325     "pmaddwd                   %%mm4, %%mm3         \n\t"\
326     "paddd                     %%mm0, %%mm1         \n\t"\
327     "paddd                     %%mm3, %%mm5         \n\t"\
328     "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3         \n\t" /* Y2srcData */\
329     "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S"     \n\t"\
330     "add           $"STR(APCK_SIZE)", %%"REG_d"     \n\t"\
331     "test                  %%"REG_S", %%"REG_S"     \n\t"\
332     "movq                      %%mm2, %%mm0         \n\t"\
333     "punpcklwd                 %%mm3, %%mm2         \n\t"\
334     "punpckhwd                 %%mm3, %%mm0         \n\t"\
335     "pmaddwd                   %%mm4, %%mm2         \n\t"\
336     "pmaddwd                   %%mm4, %%mm0         \n\t"\
337     "paddd                     %%mm2, %%mm7         \n\t"\
338     "paddd                     %%mm0, %%mm6         \n\t"\
339     " jnz                         2b                \n\t"\
340     "psrad                       $16, %%mm1         \n\t"\
341     "psrad                       $16, %%mm5         \n\t"\
342     "psrad                       $16, %%mm7         \n\t"\
343     "psrad                       $16, %%mm6         \n\t"\
344     "movq      "VROUNDER_OFFSET"(%0), %%mm0         \n\t"\
345     "packssdw                  %%mm5, %%mm1         \n\t"\
346     "packssdw                  %%mm6, %%mm7         \n\t"\
347     "paddw                     %%mm0, %%mm1         \n\t"\
348     "paddw                     %%mm0, %%mm7         \n\t"\
349     "movq               "U_TEMP"(%0), %%mm3         \n\t"\
350     "movq               "V_TEMP"(%0), %%mm4         \n\t"\
351
352 #define YSCALEYUV2RGBX \
353     "psubw  "U_OFFSET"(%0), %%mm3       \n\t" /* (U-128)8*/\
354     "psubw  "V_OFFSET"(%0), %%mm4       \n\t" /* (V-128)8*/\
355     "movq            %%mm3, %%mm2       \n\t" /* (U-128)8*/\
356     "movq            %%mm4, %%mm5       \n\t" /* (V-128)8*/\
357     "pmulhw "UG_COEFF"(%0), %%mm3       \n\t"\
358     "pmulhw "VG_COEFF"(%0), %%mm4       \n\t"\
359 /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
360     "pmulhw "UB_COEFF"(%0), %%mm2       \n\t"\
361     "pmulhw "VR_COEFF"(%0), %%mm5       \n\t"\
362     "psubw  "Y_OFFSET"(%0), %%mm1       \n\t" /* 8(Y-16)*/\
363     "psubw  "Y_OFFSET"(%0), %%mm7       \n\t" /* 8(Y-16)*/\
364     "pmulhw  "Y_COEFF"(%0), %%mm1       \n\t"\
365     "pmulhw  "Y_COEFF"(%0), %%mm7       \n\t"\
366 /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
367     "paddw           %%mm3, %%mm4       \n\t"\
368     "movq            %%mm2, %%mm0       \n\t"\
369     "movq            %%mm5, %%mm6       \n\t"\
370     "movq            %%mm4, %%mm3       \n\t"\
371     "punpcklwd       %%mm2, %%mm2       \n\t"\
372     "punpcklwd       %%mm5, %%mm5       \n\t"\
373     "punpcklwd       %%mm4, %%mm4       \n\t"\
374     "paddw           %%mm1, %%mm2       \n\t"\
375     "paddw           %%mm1, %%mm5       \n\t"\
376     "paddw           %%mm1, %%mm4       \n\t"\
377     "punpckhwd       %%mm0, %%mm0       \n\t"\
378     "punpckhwd       %%mm6, %%mm6       \n\t"\
379     "punpckhwd       %%mm3, %%mm3       \n\t"\
380     "paddw           %%mm7, %%mm0       \n\t"\
381     "paddw           %%mm7, %%mm6       \n\t"\
382     "paddw           %%mm7, %%mm3       \n\t"\
383     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
384     "packuswb        %%mm0, %%mm2       \n\t"\
385     "packuswb        %%mm6, %%mm5       \n\t"\
386     "packuswb        %%mm3, %%mm4       \n\t"\
387     "pxor            %%mm7, %%mm7       \n\t"
388
389 #define REAL_YSCALEYUV2PACKED(index, c) \
390     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0              \n\t"\
391     "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1              \n\t"\
392     "psraw                $3, %%mm0                           \n\t"\
393     "psraw                $3, %%mm1                           \n\t"\
394     "movq              %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
395     "movq              %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
396     "xor            "#index", "#index"                        \n\t"\
397     ASMALIGN(4)\
398     "1:                                 \n\t"\
399     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
400     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
401     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
402     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
403     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
404     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
405     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
406     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
407     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
408     "psraw                $7, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
409     "psraw                $7, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
410     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
411     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
412     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
413     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
414     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
415     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
416     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
417     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
418     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
419     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
420     "psraw                $7, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
421     "psraw                $7, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
422     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
423     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
424
425 #define YSCALEYUV2PACKED(index, c)  REAL_YSCALEYUV2PACKED(index, c)
426
427 #define REAL_YSCALEYUV2RGB(index, c) \
428     "xor            "#index", "#index"  \n\t"\
429     ASMALIGN(4)\
430     "1:                                 \n\t"\
431     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
432     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
433     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
434     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
435     "psubw             %%mm3, %%mm2     \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
436     "psubw             %%mm4, %%mm5     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
437     "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0    \n\t"\
438     "pmulhw            %%mm0, %%mm2     \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
439     "pmulhw            %%mm0, %%mm5     \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
440     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
441     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
442     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
443     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
444     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
445     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
446     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
447     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
448     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
449     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
450     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
451     "movq  (%0, "#index", 2), %%mm0     \n\t" /*buf0[eax]*/\
452     "movq  (%1, "#index", 2), %%mm1     \n\t" /*buf1[eax]*/\
453     "movq 8(%0, "#index", 2), %%mm6     \n\t" /*buf0[eax]*/\
454     "movq 8(%1, "#index", 2), %%mm7     \n\t" /*buf1[eax]*/\
455     "psubw             %%mm1, %%mm0     \n\t" /* buf0[eax] - buf1[eax]*/\
456     "psubw             %%mm7, %%mm6     \n\t" /* buf0[eax] - buf1[eax]*/\
457     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
458     "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6  \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
459     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
460     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
461     "paddw             %%mm0, %%mm1     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
462     "paddw             %%mm6, %%mm7     \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
463     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
464     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
465     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
466     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
467     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
468     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
469     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
470     "paddw             %%mm3, %%mm4     \n\t"\
471     "movq              %%mm2, %%mm0     \n\t"\
472     "movq              %%mm5, %%mm6     \n\t"\
473     "movq              %%mm4, %%mm3     \n\t"\
474     "punpcklwd         %%mm2, %%mm2     \n\t"\
475     "punpcklwd         %%mm5, %%mm5     \n\t"\
476     "punpcklwd         %%mm4, %%mm4     \n\t"\
477     "paddw             %%mm1, %%mm2     \n\t"\
478     "paddw             %%mm1, %%mm5     \n\t"\
479     "paddw             %%mm1, %%mm4     \n\t"\
480     "punpckhwd         %%mm0, %%mm0     \n\t"\
481     "punpckhwd         %%mm6, %%mm6     \n\t"\
482     "punpckhwd         %%mm3, %%mm3     \n\t"\
483     "paddw             %%mm7, %%mm0     \n\t"\
484     "paddw             %%mm7, %%mm6     \n\t"\
485     "paddw             %%mm7, %%mm3     \n\t"\
486     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
487     "packuswb          %%mm0, %%mm2     \n\t"\
488     "packuswb          %%mm6, %%mm5     \n\t"\
489     "packuswb          %%mm3, %%mm4     \n\t"\
490     "pxor              %%mm7, %%mm7     \n\t"
491 #define YSCALEYUV2RGB(index, c)  REAL_YSCALEYUV2RGB(index, c)
492
493 #define REAL_YSCALEYUV2PACKED1(index, c) \
494     "xor            "#index", "#index"  \n\t"\
495     ASMALIGN(4)\
496     "1:                                 \n\t"\
497     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
498     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
499     "psraw                $7, %%mm3     \n\t" \
500     "psraw                $7, %%mm4     \n\t" \
501     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
502     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
503     "psraw                $7, %%mm1     \n\t" \
504     "psraw                $7, %%mm7     \n\t" \
505
506 #define YSCALEYUV2PACKED1(index, c)  REAL_YSCALEYUV2PACKED1(index, c)
507
508 #define REAL_YSCALEYUV2RGB1(index, c) \
509     "xor            "#index", "#index"  \n\t"\
510     ASMALIGN(4)\
511     "1:                                 \n\t"\
512     "movq     (%2, "#index"), %%mm3     \n\t" /* uvbuf0[eax]*/\
513     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4     \n\t" /* uvbuf0[eax+2048]*/\
514     "psraw                $4, %%mm3     \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
515     "psraw                $4, %%mm4     \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
516     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
517     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
518     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
519     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
520     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
521     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
522     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
523     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
524     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
525     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
526     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
527     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
528     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
529     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
530     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
531     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
532     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
533     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
534     "paddw             %%mm3, %%mm4     \n\t"\
535     "movq              %%mm2, %%mm0     \n\t"\
536     "movq              %%mm5, %%mm6     \n\t"\
537     "movq              %%mm4, %%mm3     \n\t"\
538     "punpcklwd         %%mm2, %%mm2     \n\t"\
539     "punpcklwd         %%mm5, %%mm5     \n\t"\
540     "punpcklwd         %%mm4, %%mm4     \n\t"\
541     "paddw             %%mm1, %%mm2     \n\t"\
542     "paddw             %%mm1, %%mm5     \n\t"\
543     "paddw             %%mm1, %%mm4     \n\t"\
544     "punpckhwd         %%mm0, %%mm0     \n\t"\
545     "punpckhwd         %%mm6, %%mm6     \n\t"\
546     "punpckhwd         %%mm3, %%mm3     \n\t"\
547     "paddw             %%mm7, %%mm0     \n\t"\
548     "paddw             %%mm7, %%mm6     \n\t"\
549     "paddw             %%mm7, %%mm3     \n\t"\
550     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
551     "packuswb          %%mm0, %%mm2     \n\t"\
552     "packuswb          %%mm6, %%mm5     \n\t"\
553     "packuswb          %%mm3, %%mm4     \n\t"\
554     "pxor              %%mm7, %%mm7     \n\t"
555 #define YSCALEYUV2RGB1(index, c)  REAL_YSCALEYUV2RGB1(index, c)
556
557 #define REAL_YSCALEYUV2PACKED1b(index, c) \
558     "xor "#index", "#index"             \n\t"\
559     ASMALIGN(4)\
560     "1:                                 \n\t"\
561     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
562     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
563     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
564     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
565     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
566     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
567     "psrlw                $8, %%mm3     \n\t" \
568     "psrlw                $8, %%mm4     \n\t" \
569     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
570     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
571     "psraw                $7, %%mm1     \n\t" \
572     "psraw                $7, %%mm7     \n\t"
573 #define YSCALEYUV2PACKED1b(index, c)  REAL_YSCALEYUV2PACKED1b(index, c)
574
575 // do vertical chrominance interpolation
576 #define REAL_YSCALEYUV2RGB1b(index, c) \
577     "xor            "#index", "#index"  \n\t"\
578     ASMALIGN(4)\
579     "1:                                 \n\t"\
580     "movq     (%2, "#index"), %%mm2     \n\t" /* uvbuf0[eax]*/\
581     "movq     (%3, "#index"), %%mm3     \n\t" /* uvbuf1[eax]*/\
582     "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5     \n\t" /* uvbuf0[eax+2048]*/\
583     "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4     \n\t" /* uvbuf1[eax+2048]*/\
584     "paddw             %%mm2, %%mm3     \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
585     "paddw             %%mm5, %%mm4     \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
586     "psrlw                $5, %%mm3     \n\t" /*FIXME might overflow*/\
587     "psrlw                $5, %%mm4     \n\t" /*FIXME might overflow*/\
588     "psubw  "U_OFFSET"("#c"), %%mm3     \n\t" /* (U-128)8*/\
589     "psubw  "V_OFFSET"("#c"), %%mm4     \n\t" /* (V-128)8*/\
590     "movq              %%mm3, %%mm2     \n\t" /* (U-128)8*/\
591     "movq              %%mm4, %%mm5     \n\t" /* (V-128)8*/\
592     "pmulhw "UG_COEFF"("#c"), %%mm3     \n\t"\
593     "pmulhw "VG_COEFF"("#c"), %%mm4     \n\t"\
594     /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
595     "movq  (%0, "#index", 2), %%mm1     \n\t" /*buf0[eax]*/\
596     "movq 8(%0, "#index", 2), %%mm7     \n\t" /*buf0[eax]*/\
597     "psraw                $4, %%mm1     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
598     "psraw                $4, %%mm7     \n\t" /* buf0[eax] - buf1[eax] >>4*/\
599     "pmulhw "UB_COEFF"("#c"), %%mm2     \n\t"\
600     "pmulhw "VR_COEFF"("#c"), %%mm5     \n\t"\
601     "psubw  "Y_OFFSET"("#c"), %%mm1     \n\t" /* 8(Y-16)*/\
602     "psubw  "Y_OFFSET"("#c"), %%mm7     \n\t" /* 8(Y-16)*/\
603     "pmulhw  "Y_COEFF"("#c"), %%mm1     \n\t"\
604     "pmulhw  "Y_COEFF"("#c"), %%mm7     \n\t"\
605     /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
606     "paddw             %%mm3, %%mm4     \n\t"\
607     "movq              %%mm2, %%mm0     \n\t"\
608     "movq              %%mm5, %%mm6     \n\t"\
609     "movq              %%mm4, %%mm3     \n\t"\
610     "punpcklwd         %%mm2, %%mm2     \n\t"\
611     "punpcklwd         %%mm5, %%mm5     \n\t"\
612     "punpcklwd         %%mm4, %%mm4     \n\t"\
613     "paddw             %%mm1, %%mm2     \n\t"\
614     "paddw             %%mm1, %%mm5     \n\t"\
615     "paddw             %%mm1, %%mm4     \n\t"\
616     "punpckhwd         %%mm0, %%mm0     \n\t"\
617     "punpckhwd         %%mm6, %%mm6     \n\t"\
618     "punpckhwd         %%mm3, %%mm3     \n\t"\
619     "paddw             %%mm7, %%mm0     \n\t"\
620     "paddw             %%mm7, %%mm6     \n\t"\
621     "paddw             %%mm7, %%mm3     \n\t"\
622     /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
623     "packuswb          %%mm0, %%mm2     \n\t"\
624     "packuswb          %%mm6, %%mm5     \n\t"\
625     "packuswb          %%mm3, %%mm4     \n\t"\
626     "pxor              %%mm7, %%mm7     \n\t"
627 #define YSCALEYUV2RGB1b(index, c)  REAL_YSCALEYUV2RGB1b(index, c)
628
629 #define REAL_WRITEBGR32(dst, dstw, index) \
630     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
631     "movq      %%mm2, %%mm1     \n\t" /* B */\
632     "movq      %%mm5, %%mm6     \n\t" /* R */\
633     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
634     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
635     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
636     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
637     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
638     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
639     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
640     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
641     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
642     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
643 \
644     MOVNTQ(%%mm0,   (dst, index, 4))\
645     MOVNTQ(%%mm2,  8(dst, index, 4))\
646     MOVNTQ(%%mm1, 16(dst, index, 4))\
647     MOVNTQ(%%mm3, 24(dst, index, 4))\
648 \
649     "add      $8, "#index"      \n\t"\
650     "cmp "#dstw", "#index"      \n\t"\
651     " jb      1b                \n\t"
652 #define WRITEBGR32(dst, dstw, index)  REAL_WRITEBGR32(dst, dstw, index)
653
654 #define REAL_WRITERGB16(dst, dstw, index) \
655     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
656     "pand "MANGLE(bFC)", %%mm4  \n\t" /* G */\
657     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
658     "psrlq           $3, %%mm2  \n\t"\
659 \
660     "movq         %%mm2, %%mm1  \n\t"\
661     "movq         %%mm4, %%mm3  \n\t"\
662 \
663     "punpcklbw    %%mm7, %%mm3  \n\t"\
664     "punpcklbw    %%mm5, %%mm2  \n\t"\
665     "punpckhbw    %%mm7, %%mm4  \n\t"\
666     "punpckhbw    %%mm5, %%mm1  \n\t"\
667 \
668     "psllq           $3, %%mm3  \n\t"\
669     "psllq           $3, %%mm4  \n\t"\
670 \
671     "por          %%mm3, %%mm2  \n\t"\
672     "por          %%mm4, %%mm1  \n\t"\
673 \
674     MOVNTQ(%%mm2,  (dst, index, 2))\
675     MOVNTQ(%%mm1, 8(dst, index, 2))\
676 \
677     "add             $8, "#index"   \n\t"\
678     "cmp        "#dstw", "#index"   \n\t"\
679     " jb             1b             \n\t"
680 #define WRITERGB16(dst, dstw, index)  REAL_WRITERGB16(dst, dstw, index)
681
682 #define REAL_WRITERGB15(dst, dstw, index) \
683     "pand "MANGLE(bF8)", %%mm2  \n\t" /* B */\
684     "pand "MANGLE(bF8)", %%mm4  \n\t" /* G */\
685     "pand "MANGLE(bF8)", %%mm5  \n\t" /* R */\
686     "psrlq           $3, %%mm2  \n\t"\
687     "psrlq           $1, %%mm5  \n\t"\
688 \
689     "movq         %%mm2, %%mm1  \n\t"\
690     "movq         %%mm4, %%mm3  \n\t"\
691 \
692     "punpcklbw    %%mm7, %%mm3  \n\t"\
693     "punpcklbw    %%mm5, %%mm2  \n\t"\
694     "punpckhbw    %%mm7, %%mm4  \n\t"\
695     "punpckhbw    %%mm5, %%mm1  \n\t"\
696 \
697     "psllq           $2, %%mm3  \n\t"\
698     "psllq           $2, %%mm4  \n\t"\
699 \
700     "por          %%mm3, %%mm2  \n\t"\
701     "por          %%mm4, %%mm1  \n\t"\
702 \
703     MOVNTQ(%%mm2,  (dst, index, 2))\
704     MOVNTQ(%%mm1, 8(dst, index, 2))\
705 \
706     "add             $8, "#index"   \n\t"\
707     "cmp        "#dstw", "#index"   \n\t"\
708     " jb             1b             \n\t"
709 #define WRITERGB15(dst, dstw, index)  REAL_WRITERGB15(dst, dstw, index)
710
711 #define WRITEBGR24OLD(dst, dstw, index) \
712     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
713     "movq      %%mm2, %%mm1             \n\t" /* B */\
714     "movq      %%mm5, %%mm6             \n\t" /* R */\
715     "punpcklbw %%mm4, %%mm2             \n\t" /* GBGBGBGB 0 */\
716     "punpcklbw %%mm7, %%mm5             \n\t" /* 0R0R0R0R 0 */\
717     "punpckhbw %%mm4, %%mm1             \n\t" /* GBGBGBGB 2 */\
718     "punpckhbw %%mm7, %%mm6             \n\t" /* 0R0R0R0R 2 */\
719     "movq      %%mm2, %%mm0             \n\t" /* GBGBGBGB 0 */\
720     "movq      %%mm1, %%mm3             \n\t" /* GBGBGBGB 2 */\
721     "punpcklwd %%mm5, %%mm0             \n\t" /* 0RGB0RGB 0 */\
722     "punpckhwd %%mm5, %%mm2             \n\t" /* 0RGB0RGB 1 */\
723     "punpcklwd %%mm6, %%mm1             \n\t" /* 0RGB0RGB 2 */\
724     "punpckhwd %%mm6, %%mm3             \n\t" /* 0RGB0RGB 3 */\
725 \
726     "movq      %%mm0, %%mm4             \n\t" /* 0RGB0RGB 0 */\
727     "psrlq        $8, %%mm0             \n\t" /* 00RGB0RG 0 */\
728     "pand "MANGLE(bm00000111)", %%mm4   \n\t" /* 00000RGB 0 */\
729     "pand "MANGLE(bm11111000)", %%mm0   \n\t" /* 00RGB000 0.5 */\
730     "por       %%mm4, %%mm0             \n\t" /* 00RGBRGB 0 */\
731     "movq      %%mm2, %%mm4             \n\t" /* 0RGB0RGB 1 */\
732     "psllq       $48, %%mm2             \n\t" /* GB000000 1 */\
733     "por       %%mm2, %%mm0             \n\t" /* GBRGBRGB 0 */\
734 \
735     "movq      %%mm4, %%mm2             \n\t" /* 0RGB0RGB 1 */\
736     "psrld       $16, %%mm4             \n\t" /* 000R000R 1 */\
737     "psrlq       $24, %%mm2             \n\t" /* 0000RGB0 1.5 */\
738     "por       %%mm4, %%mm2             \n\t" /* 000RRGBR 1 */\
739     "pand "MANGLE(bm00001111)", %%mm2   \n\t" /* 0000RGBR 1 */\
740     "movq      %%mm1, %%mm4             \n\t" /* 0RGB0RGB 2 */\
741     "psrlq        $8, %%mm1             \n\t" /* 00RGB0RG 2 */\
742     "pand "MANGLE(bm00000111)", %%mm4   \n\t" /* 00000RGB 2 */\
743     "pand "MANGLE(bm11111000)", %%mm1   \n\t" /* 00RGB000 2.5 */\
744     "por       %%mm4, %%mm1             \n\t" /* 00RGBRGB 2 */\
745     "movq      %%mm1, %%mm4             \n\t" /* 00RGBRGB 2 */\
746     "psllq       $32, %%mm1             \n\t" /* BRGB0000 2 */\
747     "por       %%mm1, %%mm2             \n\t" /* BRGBRGBR 1 */\
748 \
749     "psrlq       $32, %%mm4             \n\t" /* 000000RG 2.5 */\
750     "movq      %%mm3, %%mm5             \n\t" /* 0RGB0RGB 3 */\
751     "psrlq        $8, %%mm3             \n\t" /* 00RGB0RG 3 */\
752     "pand "MANGLE(bm00000111)", %%mm5   \n\t" /* 00000RGB 3 */\
753     "pand "MANGLE(bm11111000)", %%mm3   \n\t" /* 00RGB000 3.5 */\
754     "por       %%mm5, %%mm3             \n\t" /* 00RGBRGB 3 */\
755     "psllq       $16, %%mm3             \n\t" /* RGBRGB00 3 */\
756     "por       %%mm4, %%mm3             \n\t" /* RGBRGBRG 2.5 */\
757 \
758     MOVNTQ(%%mm0,   (dst))\
759     MOVNTQ(%%mm2,  8(dst))\
760     MOVNTQ(%%mm3, 16(dst))\
761     "add         $24, "#dst"            \n\t"\
762 \
763     "add          $8, "#index"          \n\t"\
764     "cmp     "#dstw", "#index"          \n\t"\
765     " jb          1b                    \n\t"
766
767 #define WRITEBGR24MMX(dst, dstw, index) \
768     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
769     "movq      %%mm2, %%mm1     \n\t" /* B */\
770     "movq      %%mm5, %%mm6     \n\t" /* R */\
771     "punpcklbw %%mm4, %%mm2     \n\t" /* GBGBGBGB 0 */\
772     "punpcklbw %%mm7, %%mm5     \n\t" /* 0R0R0R0R 0 */\
773     "punpckhbw %%mm4, %%mm1     \n\t" /* GBGBGBGB 2 */\
774     "punpckhbw %%mm7, %%mm6     \n\t" /* 0R0R0R0R 2 */\
775     "movq      %%mm2, %%mm0     \n\t" /* GBGBGBGB 0 */\
776     "movq      %%mm1, %%mm3     \n\t" /* GBGBGBGB 2 */\
777     "punpcklwd %%mm5, %%mm0     \n\t" /* 0RGB0RGB 0 */\
778     "punpckhwd %%mm5, %%mm2     \n\t" /* 0RGB0RGB 1 */\
779     "punpcklwd %%mm6, %%mm1     \n\t" /* 0RGB0RGB 2 */\
780     "punpckhwd %%mm6, %%mm3     \n\t" /* 0RGB0RGB 3 */\
781 \
782     "movq      %%mm0, %%mm4     \n\t" /* 0RGB0RGB 0 */\
783     "movq      %%mm2, %%mm6     \n\t" /* 0RGB0RGB 1 */\
784     "movq      %%mm1, %%mm5     \n\t" /* 0RGB0RGB 2 */\
785     "movq      %%mm3, %%mm7     \n\t" /* 0RGB0RGB 3 */\
786 \
787     "psllq       $40, %%mm0     \n\t" /* RGB00000 0 */\
788     "psllq       $40, %%mm2     \n\t" /* RGB00000 1 */\
789     "psllq       $40, %%mm1     \n\t" /* RGB00000 2 */\
790     "psllq       $40, %%mm3     \n\t" /* RGB00000 3 */\
791 \
792     "punpckhdq %%mm4, %%mm0     \n\t" /* 0RGBRGB0 0 */\
793     "punpckhdq %%mm6, %%mm2     \n\t" /* 0RGBRGB0 1 */\
794     "punpckhdq %%mm5, %%mm1     \n\t" /* 0RGBRGB0 2 */\
795     "punpckhdq %%mm7, %%mm3     \n\t" /* 0RGBRGB0 3 */\
796 \
797     "psrlq        $8, %%mm0     \n\t" /* 00RGBRGB 0 */\
798     "movq      %%mm2, %%mm6     \n\t" /* 0RGBRGB0 1 */\
799     "psllq       $40, %%mm2     \n\t" /* GB000000 1 */\
800     "por       %%mm2, %%mm0     \n\t" /* GBRGBRGB 0 */\
801     MOVNTQ(%%mm0, (dst))\
802 \
803     "psrlq       $24, %%mm6     \n\t" /* 0000RGBR 1 */\
804     "movq      %%mm1, %%mm5     \n\t" /* 0RGBRGB0 2 */\
805     "psllq       $24, %%mm1     \n\t" /* BRGB0000 2 */\
806     "por       %%mm1, %%mm6     \n\t" /* BRGBRGBR 1 */\
807     MOVNTQ(%%mm6, 8(dst))\
808 \
809     "psrlq       $40, %%mm5     \n\t" /* 000000RG 2 */\
810     "psllq        $8, %%mm3     \n\t" /* RGBRGB00 3 */\
811     "por       %%mm3, %%mm5     \n\t" /* RGBRGBRG 2 */\
812     MOVNTQ(%%mm5, 16(dst))\
813 \
814     "add         $24, "#dst"    \n\t"\
815 \
816     "add          $8, "#index"  \n\t"\
817     "cmp     "#dstw", "#index"  \n\t"\
818     " jb          1b            \n\t"
819
820 #define WRITEBGR24MMX2(dst, dstw, index) \
821     /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
822     "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
823     "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
824     "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2  B1 B0 B1 B0 */\
825     "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2  G1 G0 G1 G0 */\
826     "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0  R1 R0 R1 R0 */\
827 \
828     "pand   %%mm0, %%mm1        \n\t" /*    B2        B1       B0 */\
829     "pand   %%mm0, %%mm3        \n\t" /*    G2        G1       G0 */\
830     "pand   %%mm7, %%mm6        \n\t" /*       R1        R0       */\
831 \
832     "psllq     $8, %%mm3        \n\t" /* G2        G1       G0    */\
833     "por    %%mm1, %%mm6        \n\t"\
834     "por    %%mm3, %%mm6        \n\t"\
835     MOVNTQ(%%mm6, (dst))\
836 \
837     "psrlq     $8, %%mm4        \n\t" /* 00 G7 G6 G5  G4 G3 G2 G1 */\
838     "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4  B3 B2 B3 B2 */\
839     "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3  G4 G3 G4 G3 */\
840     "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4  R3 R2 R3 R2 */\
841 \
842     "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5       B4        B3    */\
843     "pand   %%mm7, %%mm3        \n\t" /*       G4        G3       */\
844     "pand   %%mm0, %%mm6        \n\t" /*    R4        R3       R2 */\
845 \
846     "por    %%mm1, %%mm3        \n\t" /* B5    G4 B4     G3 B3    */\
847     "por    %%mm3, %%mm6        \n\t"\
848     MOVNTQ(%%mm6, 8(dst))\
849 \
850     "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6  B7 B6 B6 B7 */\
851     "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7  G6 G5 G6 G5 */\
852     "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6  R5 R4 R5 R4 */\
853 \
854     "pand   %%mm7, %%mm1        \n\t" /*       B7        B6       */\
855     "pand   %%mm0, %%mm3        \n\t" /*    G7        G6       G5 */\
856     "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7       R6        R5    */\
857 \
858     "por    %%mm1, %%mm3        \n\t"\
859     "por    %%mm3, %%mm6        \n\t"\
860     MOVNTQ(%%mm6, 16(dst))\
861 \
862     "add      $24, "#dst"       \n\t"\
863 \
864     "add       $8, "#index"     \n\t"\
865     "cmp  "#dstw", "#index"     \n\t"\
866     " jb       1b               \n\t"
867
868 #ifdef HAVE_MMX2
869 #undef WRITEBGR24
870 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX2(dst, dstw, index)
871 #else
872 #undef WRITEBGR24
873 #define WRITEBGR24(dst, dstw, index)  WRITEBGR24MMX(dst, dstw, index)
874 #endif
875
876 #define REAL_WRITEYUY2(dst, dstw, index) \
877     "packuswb  %%mm3, %%mm3     \n\t"\
878     "packuswb  %%mm4, %%mm4     \n\t"\
879     "packuswb  %%mm7, %%mm1     \n\t"\
880     "punpcklbw %%mm4, %%mm3     \n\t"\
881     "movq      %%mm1, %%mm7     \n\t"\
882     "punpcklbw %%mm3, %%mm1     \n\t"\
883     "punpckhbw %%mm3, %%mm7     \n\t"\
884 \
885     MOVNTQ(%%mm1, (dst, index, 2))\
886     MOVNTQ(%%mm7, 8(dst, index, 2))\
887 \
888     "add          $8, "#index"  \n\t"\
889     "cmp     "#dstw", "#index"  \n\t"\
890     " jb          1b            \n\t"
891 #define WRITEYUY2(dst, dstw, index)  REAL_WRITEYUY2(dst, dstw, index)
892
893
894 static inline void RENAME(yuv2yuvX)(SwsContext *c, int16_t *lumFilter, int16_t **lumSrc, int lumFilterSize,
895                                     int16_t *chrFilter, int16_t **chrSrc, int chrFilterSize,
896                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest, long dstW, long chrDstW)
897 {
898 #ifdef HAVE_MMX
899     if(!(c->flags & SWS_BITEXACT)){
900         if (c->flags & SWS_ACCURATE_RND){
901             if (uDest){
902                 YSCALEYUV2YV12X_ACCURATE(   "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
903                 YSCALEYUV2YV12X_ACCURATE(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
904             }
905
906             YSCALEYUV2YV12X_ACCURATE("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
907         }else{
908             if (uDest){
909                 YSCALEYUV2YV12X(   "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
910                 YSCALEYUV2YV12X(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
911             }
912
913             YSCALEYUV2YV12X("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
914         }
915         return;
916     }
917 #endif
918 #ifdef HAVE_ALTIVEC
919 yuv2yuvX_altivec_real(lumFilter, lumSrc, lumFilterSize,
920                       chrFilter, chrSrc, chrFilterSize,
921                       dest, uDest, vDest, dstW, chrDstW);
922 #else //HAVE_ALTIVEC
923 yuv2yuvXinC(lumFilter, lumSrc, lumFilterSize,
924             chrFilter, chrSrc, chrFilterSize,
925             dest, uDest, vDest, dstW, chrDstW);
926 #endif //!HAVE_ALTIVEC
927 }
928
929 static inline void RENAME(yuv2nv12X)(SwsContext *c, int16_t *lumFilter, int16_t **lumSrc, int lumFilterSize,
930                                      int16_t *chrFilter, int16_t **chrSrc, int chrFilterSize,
931                                      uint8_t *dest, uint8_t *uDest, int dstW, int chrDstW, int dstFormat)
932 {
933 yuv2nv12XinC(lumFilter, lumSrc, lumFilterSize,
934              chrFilter, chrSrc, chrFilterSize,
935              dest, uDest, dstW, chrDstW, dstFormat);
936 }
937
938 static inline void RENAME(yuv2yuv1)(SwsContext *c, int16_t *lumSrc, int16_t *chrSrc,
939                                     uint8_t *dest, uint8_t *uDest, uint8_t *vDest, long dstW, long chrDstW)
940 {
941     int i;
942 #ifdef HAVE_MMX
943     if(!(c->flags & SWS_BITEXACT)){
944         long p= uDest ? 3 : 1;
945         uint8_t *src[3]= {lumSrc + dstW, chrSrc + chrDstW, chrSrc + VOFW + chrDstW};
946         uint8_t *dst[3]= {dest, uDest, vDest};
947         long counter[3] = {dstW, chrDstW, chrDstW};
948
949         if (c->flags & SWS_ACCURATE_RND){
950             while(p--){
951                 __asm__ volatile(
952                     YSCALEYUV2YV121_ACCURATE
953                     :: "r" (src[p]), "r" (dst[p] + counter[p]),
954                     "g" (-counter[p])
955                     : "%"REG_a
956                 );
957             }
958         }else{
959             while(p--){
960                 __asm__ volatile(
961                     YSCALEYUV2YV121
962                     :: "r" (src[p]), "r" (dst[p] + counter[p]),
963                     "g" (-counter[p])
964                     : "%"REG_a
965                 );
966             }
967         }
968         return;
969     }
970 #endif
971     for (i=0; i<dstW; i++)
972     {
973         int val= (lumSrc[i]+64)>>7;
974
975         if (val&256){
976             if (val<0) val=0;
977             else       val=255;
978         }
979
980         dest[i]= val;
981     }
982
983     if (uDest)
984         for (i=0; i<chrDstW; i++)
985         {
986             int u=(chrSrc[i       ]+64)>>7;
987             int v=(chrSrc[i + VOFW]+64)>>7;
988
989             if ((u|v)&256){
990                 if (u<0)        u=0;
991                 else if (u>255) u=255;
992                 if (v<0)        v=0;
993                 else if (v>255) v=255;
994             }
995
996             uDest[i]= u;
997             vDest[i]= v;
998         }
999 }
1000
1001
1002 /**
1003  * vertical scale YV12 to RGB
1004  */
1005 static inline void RENAME(yuv2packedX)(SwsContext *c, int16_t *lumFilter, int16_t **lumSrc, int lumFilterSize,
1006                                        int16_t *chrFilter, int16_t **chrSrc, int chrFilterSize,
1007                                        uint8_t *dest, long dstW, long dstY)
1008 {
1009 #ifdef HAVE_MMX
1010     long dummy=0;
1011     if(!(c->flags & SWS_BITEXACT)){
1012         if (c->flags & SWS_ACCURATE_RND){
1013             switch(c->dstFormat){
1014             case PIX_FMT_RGB32:
1015                 YSCALEYUV2PACKEDX_ACCURATE
1016                 YSCALEYUV2RGBX
1017                 WRITEBGR32(%4, %5, %%REGa)
1018
1019                 YSCALEYUV2PACKEDX_END
1020                 return;
1021             case PIX_FMT_BGR24:
1022                 YSCALEYUV2PACKEDX_ACCURATE
1023                 YSCALEYUV2RGBX
1024                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
1025                 "add %4, %%"REG_c"                        \n\t"
1026                 WRITEBGR24(%%REGc, %5, %%REGa)
1027
1028
1029                 :: "r" (&c->redDither),
1030                 "m" (dummy), "m" (dummy), "m" (dummy),
1031                 "r" (dest), "m" (dstW)
1032                 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
1033                 );
1034                 return;
1035             case PIX_FMT_RGB555:
1036                 YSCALEYUV2PACKEDX_ACCURATE
1037                 YSCALEYUV2RGBX
1038                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1039 #ifdef DITHER1XBPP
1040                 "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
1041                 "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
1042                 "paddusb "RED_DITHER"(%0), %%mm5\n\t"
1043 #endif
1044
1045                 WRITERGB15(%4, %5, %%REGa)
1046                 YSCALEYUV2PACKEDX_END
1047                 return;
1048             case PIX_FMT_RGB565:
1049                 YSCALEYUV2PACKEDX_ACCURATE
1050                 YSCALEYUV2RGBX
1051                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1052 #ifdef DITHER1XBPP
1053                 "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
1054                 "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
1055                 "paddusb "RED_DITHER"(%0), %%mm5\n\t"
1056 #endif
1057
1058                 WRITERGB16(%4, %5, %%REGa)
1059                 YSCALEYUV2PACKEDX_END
1060                 return;
1061             case PIX_FMT_YUYV422:
1062                 YSCALEYUV2PACKEDX_ACCURATE
1063                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1064
1065                 "psraw $3, %%mm3    \n\t"
1066                 "psraw $3, %%mm4    \n\t"
1067                 "psraw $3, %%mm1    \n\t"
1068                 "psraw $3, %%mm7    \n\t"
1069                 WRITEYUY2(%4, %5, %%REGa)
1070                 YSCALEYUV2PACKEDX_END
1071                 return;
1072             }
1073         }else{
1074             switch(c->dstFormat)
1075             {
1076             case PIX_FMT_RGB32:
1077                 YSCALEYUV2PACKEDX
1078                 YSCALEYUV2RGBX
1079                 WRITEBGR32(%4, %5, %%REGa)
1080                 YSCALEYUV2PACKEDX_END
1081                 return;
1082             case PIX_FMT_BGR24:
1083                 YSCALEYUV2PACKEDX
1084                 YSCALEYUV2RGBX
1085                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"   \n\t" //FIXME optimize
1086                 "add                        %4, %%"REG_c"   \n\t"
1087                 WRITEBGR24(%%REGc, %5, %%REGa)
1088
1089                 :: "r" (&c->redDither),
1090                 "m" (dummy), "m" (dummy), "m" (dummy),
1091                 "r" (dest),  "m" (dstW)
1092                 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
1093                 );
1094                 return;
1095             case PIX_FMT_RGB555:
1096                 YSCALEYUV2PACKEDX
1097                 YSCALEYUV2RGBX
1098                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1099 #ifdef DITHER1XBPP
1100                 "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
1101                 "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
1102                 "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
1103 #endif
1104
1105                 WRITERGB15(%4, %5, %%REGa)
1106                 YSCALEYUV2PACKEDX_END
1107                 return;
1108             case PIX_FMT_RGB565:
1109                 YSCALEYUV2PACKEDX
1110                 YSCALEYUV2RGBX
1111                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1112 #ifdef DITHER1XBPP
1113                 "paddusb "BLUE_DITHER"(%0), %%mm2  \n\t"
1114                 "paddusb "GREEN_DITHER"(%0), %%mm4  \n\t"
1115                 "paddusb "RED_DITHER"(%0), %%mm5  \n\t"
1116 #endif
1117
1118                 WRITERGB16(%4, %5, %%REGa)
1119                 YSCALEYUV2PACKEDX_END
1120                 return;
1121             case PIX_FMT_YUYV422:
1122                 YSCALEYUV2PACKEDX
1123                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1124
1125                 "psraw $3, %%mm3    \n\t"
1126                 "psraw $3, %%mm4    \n\t"
1127                 "psraw $3, %%mm1    \n\t"
1128                 "psraw $3, %%mm7    \n\t"
1129                 WRITEYUY2(%4, %5, %%REGa)
1130                 YSCALEYUV2PACKEDX_END
1131                 return;
1132             }
1133         }
1134     }
1135 #endif /* HAVE_MMX */
1136 #ifdef HAVE_ALTIVEC
1137     /* The following list of supported dstFormat values should
1138        match what's found in the body of altivec_yuv2packedX() */
1139     if (c->dstFormat==PIX_FMT_ABGR  || c->dstFormat==PIX_FMT_BGRA  ||
1140         c->dstFormat==PIX_FMT_BGR24 || c->dstFormat==PIX_FMT_RGB24 ||
1141         c->dstFormat==PIX_FMT_RGBA  || c->dstFormat==PIX_FMT_ARGB)
1142             altivec_yuv2packedX (c, lumFilter, lumSrc, lumFilterSize,
1143                                  chrFilter, chrSrc, chrFilterSize,
1144                                  dest, dstW, dstY);
1145     else
1146 #endif
1147         yuv2packedXinC(c, lumFilter, lumSrc, lumFilterSize,
1148                        chrFilter, chrSrc, chrFilterSize,
1149                        dest, dstW, dstY);
1150 }
1151
1152 /**
1153  * vertical bilinear scale YV12 to RGB
1154  */
1155 static inline void RENAME(yuv2packed2)(SwsContext *c, uint16_t *buf0, uint16_t *buf1, uint16_t *uvbuf0, uint16_t *uvbuf1,
1156                           uint8_t *dest, int dstW, int yalpha, int uvalpha, int y)
1157 {
1158     int  yalpha1=4095- yalpha;
1159     int uvalpha1=4095-uvalpha;
1160     int i;
1161
1162 #ifdef HAVE_MMX
1163     if(!(c->flags & SWS_BITEXACT)){
1164         switch(c->dstFormat)
1165         {
1166             //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1167             case PIX_FMT_RGB32:
1168                 __asm__ volatile(
1169                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1170                 "mov        %4, %%"REG_b"               \n\t"
1171                 "push %%"REG_BP"                        \n\t"
1172                 YSCALEYUV2RGB(%%REGBP, %5)
1173                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP)
1174                 "pop %%"REG_BP"                         \n\t"
1175                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1176
1177                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1178                 "a" (&c->redDither)
1179                 );
1180                 return;
1181             case PIX_FMT_BGR24:
1182                 __asm__ volatile(
1183                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1184                 "mov        %4, %%"REG_b"               \n\t"
1185                 "push %%"REG_BP"                        \n\t"
1186                 YSCALEYUV2RGB(%%REGBP, %5)
1187                 WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1188                 "pop %%"REG_BP"                         \n\t"
1189                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1190                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1191                 "a" (&c->redDither)
1192                 );
1193                 return;
1194             case PIX_FMT_RGB555:
1195                 __asm__ volatile(
1196                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1197                 "mov        %4, %%"REG_b"               \n\t"
1198                 "push %%"REG_BP"                        \n\t"
1199                 YSCALEYUV2RGB(%%REGBP, %5)
1200                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1201 #ifdef DITHER1XBPP
1202                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1203                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1204                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1205 #endif
1206
1207                 WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1208                 "pop %%"REG_BP"                         \n\t"
1209                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1210
1211                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1212                 "a" (&c->redDither)
1213                 );
1214                 return;
1215             case PIX_FMT_RGB565:
1216                 __asm__ volatile(
1217                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1218                 "mov        %4, %%"REG_b"               \n\t"
1219                 "push %%"REG_BP"                        \n\t"
1220                 YSCALEYUV2RGB(%%REGBP, %5)
1221                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1222 #ifdef DITHER1XBPP
1223                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1224                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1225                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1226 #endif
1227
1228                 WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1229                 "pop %%"REG_BP"                         \n\t"
1230                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1231                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1232                 "a" (&c->redDither)
1233                 );
1234                 return;
1235             case PIX_FMT_YUYV422:
1236                 __asm__ volatile(
1237                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1238                 "mov %4, %%"REG_b"                        \n\t"
1239                 "push %%"REG_BP"                        \n\t"
1240                 YSCALEYUV2PACKED(%%REGBP, %5)
1241                 WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1242                 "pop %%"REG_BP"                         \n\t"
1243                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1244                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1245                 "a" (&c->redDither)
1246                 );
1247                 return;
1248             default: break;
1249         }
1250     }
1251 #endif //HAVE_MMX
1252 YSCALE_YUV_2_ANYRGB_C(YSCALE_YUV_2_RGB2_C, YSCALE_YUV_2_PACKED2_C, YSCALE_YUV_2_GRAY16_2_C, YSCALE_YUV_2_MONO2_C)
1253 }
1254
1255 /**
1256  * YV12 to RGB without scaling or interpolating
1257  */
1258 static inline void RENAME(yuv2packed1)(SwsContext *c, uint16_t *buf0, uint16_t *uvbuf0, uint16_t *uvbuf1,
1259                           uint8_t *dest, int dstW, int uvalpha, int dstFormat, int flags, int y)
1260 {
1261     const int yalpha1=0;
1262     int i;
1263
1264     uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1265     const int yalpha= 4096; //FIXME ...
1266
1267     if (flags&SWS_FULL_CHR_H_INT)
1268     {
1269         RENAME(yuv2packed2)(c, buf0, buf0, uvbuf0, uvbuf1, dest, dstW, 0, uvalpha, y);
1270         return;
1271     }
1272
1273 #ifdef HAVE_MMX
1274     if(!(flags & SWS_BITEXACT)){
1275         if (uvalpha < 2048) // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1276         {
1277             switch(dstFormat)
1278             {
1279             case PIX_FMT_RGB32:
1280                 __asm__ volatile(
1281                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1282                 "mov        %4, %%"REG_b"               \n\t"
1283                 "push %%"REG_BP"                        \n\t"
1284                 YSCALEYUV2RGB1(%%REGBP, %5)
1285                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP)
1286                 "pop %%"REG_BP"                         \n\t"
1287                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1288
1289                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1290                 "a" (&c->redDither)
1291                 );
1292                 return;
1293             case PIX_FMT_BGR24:
1294                 __asm__ volatile(
1295                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1296                 "mov        %4, %%"REG_b"               \n\t"
1297                 "push %%"REG_BP"                        \n\t"
1298                 YSCALEYUV2RGB1(%%REGBP, %5)
1299                 WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1300                 "pop %%"REG_BP"                         \n\t"
1301                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1302
1303                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1304                 "a" (&c->redDither)
1305                 );
1306                 return;
1307             case PIX_FMT_RGB555:
1308                 __asm__ volatile(
1309                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1310                 "mov        %4, %%"REG_b"               \n\t"
1311                 "push %%"REG_BP"                        \n\t"
1312                 YSCALEYUV2RGB1(%%REGBP, %5)
1313                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1314 #ifdef DITHER1XBPP
1315                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1316                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1317                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1318 #endif
1319                 WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1320                 "pop %%"REG_BP"                         \n\t"
1321                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1322
1323                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1324                 "a" (&c->redDither)
1325                 );
1326                 return;
1327             case PIX_FMT_RGB565:
1328                 __asm__ volatile(
1329                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1330                 "mov        %4, %%"REG_b"               \n\t"
1331                 "push %%"REG_BP"                        \n\t"
1332                 YSCALEYUV2RGB1(%%REGBP, %5)
1333                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1334 #ifdef DITHER1XBPP
1335                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1336                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1337                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1338 #endif
1339
1340                 WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1341                 "pop %%"REG_BP"                         \n\t"
1342                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1343
1344                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1345                 "a" (&c->redDither)
1346                 );
1347                 return;
1348             case PIX_FMT_YUYV422:
1349                 __asm__ volatile(
1350                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1351                 "mov        %4, %%"REG_b"               \n\t"
1352                 "push %%"REG_BP"                        \n\t"
1353                 YSCALEYUV2PACKED1(%%REGBP, %5)
1354                 WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1355                 "pop %%"REG_BP"                         \n\t"
1356                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1357
1358                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1359                 "a" (&c->redDither)
1360                 );
1361                 return;
1362             }
1363         }
1364         else
1365         {
1366             switch(dstFormat)
1367             {
1368             case PIX_FMT_RGB32:
1369                 __asm__ volatile(
1370                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1371                 "mov        %4, %%"REG_b"               \n\t"
1372                 "push %%"REG_BP"                        \n\t"
1373                 YSCALEYUV2RGB1b(%%REGBP, %5)
1374                 WRITEBGR32(%%REGb, 8280(%5), %%REGBP)
1375                 "pop %%"REG_BP"                         \n\t"
1376                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1377
1378                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1379                 "a" (&c->redDither)
1380                 );
1381                 return;
1382             case PIX_FMT_BGR24:
1383                 __asm__ volatile(
1384                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1385                 "mov        %4, %%"REG_b"               \n\t"
1386                 "push %%"REG_BP"                        \n\t"
1387                 YSCALEYUV2RGB1b(%%REGBP, %5)
1388                 WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1389                 "pop %%"REG_BP"                         \n\t"
1390                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1391
1392                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1393                 "a" (&c->redDither)
1394                 );
1395                 return;
1396             case PIX_FMT_RGB555:
1397                 __asm__ volatile(
1398                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1399                 "mov        %4, %%"REG_b"               \n\t"
1400                 "push %%"REG_BP"                        \n\t"
1401                 YSCALEYUV2RGB1b(%%REGBP, %5)
1402                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1403 #ifdef DITHER1XBPP
1404                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1405                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1406                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1407 #endif
1408                 WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1409                 "pop %%"REG_BP"                         \n\t"
1410                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1411
1412                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1413                 "a" (&c->redDither)
1414                 );
1415                 return;
1416             case PIX_FMT_RGB565:
1417                 __asm__ volatile(
1418                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1419                 "mov        %4, %%"REG_b"               \n\t"
1420                 "push %%"REG_BP"                        \n\t"
1421                 YSCALEYUV2RGB1b(%%REGBP, %5)
1422                 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1423 #ifdef DITHER1XBPP
1424                 "paddusb "BLUE_DITHER"(%5), %%mm2      \n\t"
1425                 "paddusb "GREEN_DITHER"(%5), %%mm4      \n\t"
1426                 "paddusb "RED_DITHER"(%5), %%mm5      \n\t"
1427 #endif
1428
1429                 WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1430                 "pop %%"REG_BP"                         \n\t"
1431                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1432
1433                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1434                 "a" (&c->redDither)
1435                 );
1436                 return;
1437             case PIX_FMT_YUYV422:
1438                 __asm__ volatile(
1439                 "mov %%"REG_b", "ESP_OFFSET"(%5)        \n\t"
1440                 "mov        %4, %%"REG_b"               \n\t"
1441                 "push %%"REG_BP"                        \n\t"
1442                 YSCALEYUV2PACKED1b(%%REGBP, %5)
1443                 WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1444                 "pop %%"REG_BP"                         \n\t"
1445                 "mov "ESP_OFFSET"(%5), %%"REG_b"        \n\t"
1446
1447                 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1448                 "a" (&c->redDither)
1449                 );
1450                 return;
1451             }
1452         }
1453     }
1454 #endif /* HAVE_MMX */
1455     if (uvalpha < 2048)
1456     {
1457         YSCALE_YUV_2_ANYRGB_C(YSCALE_YUV_2_RGB1_C, YSCALE_YUV_2_PACKED1_C, YSCALE_YUV_2_GRAY16_1_C, YSCALE_YUV_2_MONO2_C)
1458     }else{
1459         YSCALE_YUV_2_ANYRGB_C(YSCALE_YUV_2_RGB1B_C, YSCALE_YUV_2_PACKED1B_C, YSCALE_YUV_2_GRAY16_1_C, YSCALE_YUV_2_MONO2_C)
1460     }
1461 }
1462
1463 //FIXME yuy2* can read up to 7 samples too much
1464
1465 static inline void RENAME(yuy2ToY)(uint8_t *dst, uint8_t *src, long width, uint32_t *unused)
1466 {
1467 #ifdef HAVE_MMX
1468     __asm__ volatile(
1469     "movq "MANGLE(bm01010101)", %%mm2           \n\t"
1470     "mov                    %0, %%"REG_a"       \n\t"
1471     "1:                                         \n\t"
1472     "movq    (%1, %%"REG_a",2), %%mm0           \n\t"
1473     "movq   8(%1, %%"REG_a",2), %%mm1           \n\t"
1474     "pand                %%mm2, %%mm0           \n\t"
1475     "pand                %%mm2, %%mm1           \n\t"
1476     "packuswb            %%mm1, %%mm0           \n\t"
1477     "movq                %%mm0, (%2, %%"REG_a") \n\t"
1478     "add                    $8, %%"REG_a"       \n\t"
1479     " js                    1b                  \n\t"
1480     : : "g" (-width), "r" (src+width*2), "r" (dst+width)
1481     : "%"REG_a
1482     );
1483 #else
1484     int i;
1485     for (i=0; i<width; i++)
1486         dst[i]= src[2*i];
1487 #endif
1488 }
1489
1490 static inline void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV, uint8_t *src1, uint8_t *src2, long width, uint32_t *unused)
1491 {
1492 #ifdef HAVE_MMX
1493     __asm__ volatile(
1494     "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1495     "mov                    %0, %%"REG_a"       \n\t"
1496     "1:                                         \n\t"
1497     "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1498     "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1499     "psrlw                  $8, %%mm0           \n\t"
1500     "psrlw                  $8, %%mm1           \n\t"
1501     "packuswb            %%mm1, %%mm0           \n\t"
1502     "movq                %%mm0, %%mm1           \n\t"
1503     "psrlw                  $8, %%mm0           \n\t"
1504     "pand                %%mm4, %%mm1           \n\t"
1505     "packuswb            %%mm0, %%mm0           \n\t"
1506     "packuswb            %%mm1, %%mm1           \n\t"
1507     "movd                %%mm0, (%3, %%"REG_a") \n\t"
1508     "movd                %%mm1, (%2, %%"REG_a") \n\t"
1509     "add                    $4, %%"REG_a"       \n\t"
1510     " js                    1b                  \n\t"
1511     : : "g" (-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1512     : "%"REG_a
1513     );
1514 #else
1515     int i;
1516     for (i=0; i<width; i++)
1517     {
1518         dstU[i]= src1[4*i + 1];
1519         dstV[i]= src1[4*i + 3];
1520     }
1521 #endif
1522     assert(src1 == src2);
1523 }
1524
1525 /* This is almost identical to the previous, end exists only because
1526  * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1527 static inline void RENAME(uyvyToY)(uint8_t *dst, uint8_t *src, long width, uint32_t *unused)
1528 {
1529 #ifdef HAVE_MMX
1530     __asm__ volatile(
1531     "mov                  %0, %%"REG_a"         \n\t"
1532     "1:                                         \n\t"
1533     "movq  (%1, %%"REG_a",2), %%mm0             \n\t"
1534     "movq 8(%1, %%"REG_a",2), %%mm1             \n\t"
1535     "psrlw                $8, %%mm0             \n\t"
1536     "psrlw                $8, %%mm1             \n\t"
1537     "packuswb          %%mm1, %%mm0             \n\t"
1538     "movq              %%mm0, (%2, %%"REG_a")   \n\t"
1539     "add                  $8, %%"REG_a"         \n\t"
1540     " js                  1b                    \n\t"
1541     : : "g" (-width), "r" (src+width*2), "r" (dst+width)
1542     : "%"REG_a
1543     );
1544 #else
1545     int i;
1546     for (i=0; i<width; i++)
1547         dst[i]= src[2*i+1];
1548 #endif
1549 }
1550
1551 static inline void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV, uint8_t *src1, uint8_t *src2, long width, uint32_t *unused)
1552 {
1553 #ifdef HAVE_MMX
1554     __asm__ volatile(
1555     "movq "MANGLE(bm01010101)", %%mm4           \n\t"
1556     "mov                    %0, %%"REG_a"       \n\t"
1557     "1:                                         \n\t"
1558     "movq    (%1, %%"REG_a",4), %%mm0           \n\t"
1559     "movq   8(%1, %%"REG_a",4), %%mm1           \n\t"
1560     "pand                %%mm4, %%mm0           \n\t"
1561     "pand                %%mm4, %%mm1           \n\t"
1562     "packuswb            %%mm1, %%mm0           \n\t"
1563     "movq                %%mm0, %%mm1           \n\t"
1564     "psrlw                  $8, %%mm0           \n\t"
1565     "pand                %%mm4, %%mm1           \n\t"
1566     "packuswb            %%mm0, %%mm0           \n\t"
1567     "packuswb            %%mm1, %%mm1           \n\t"
1568     "movd                %%mm0, (%3, %%"REG_a") \n\t"
1569     "movd                %%mm1, (%2, %%"REG_a") \n\t"
1570     "add                    $4, %%"REG_a"       \n\t"
1571     " js                    1b                  \n\t"
1572     : : "g" (-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1573     : "%"REG_a
1574     );
1575 #else
1576     int i;
1577     for (i=0; i<width; i++)
1578     {
1579         dstU[i]= src1[4*i + 0];
1580         dstV[i]= src1[4*i + 2];
1581     }
1582 #endif
1583     assert(src1 == src2);
1584 }
1585
1586 #define BGR2Y(type, name, shr, shg, shb, maskr, maskg, maskb, RY, GY, BY, S)\
1587 static inline void RENAME(name)(uint8_t *dst, uint8_t *src, long width, uint32_t *unused)\
1588 {\
1589     int i;\
1590     for (i=0; i<width; i++)\
1591     {\
1592         int b= (((type*)src)[i]>>shb)&maskb;\
1593         int g= (((type*)src)[i]>>shg)&maskg;\
1594         int r= (((type*)src)[i]>>shr)&maskr;\
1595 \
1596         dst[i]= (((RY)*r + (GY)*g + (BY)*b + (33<<((S)-1)))>>(S));\
1597     }\
1598 }
1599
1600 BGR2Y(uint32_t, bgr32ToY,16, 0, 0, 0x00FF, 0xFF00, 0x00FF, RY<< 8, GY   , BY<< 8, RGB2YUV_SHIFT+8)
1601 BGR2Y(uint32_t, rgb32ToY, 0, 0,16, 0x00FF, 0xFF00, 0x00FF, RY<< 8, GY   , BY<< 8, RGB2YUV_SHIFT+8)
1602 BGR2Y(uint16_t, bgr16ToY, 0, 0, 0, 0x001F, 0x07E0, 0xF800, RY<<11, GY<<5, BY    , RGB2YUV_SHIFT+8)
1603 BGR2Y(uint16_t, bgr15ToY, 0, 0, 0, 0x001F, 0x03E0, 0x7C00, RY<<10, GY<<5, BY    , RGB2YUV_SHIFT+7)
1604 BGR2Y(uint16_t, rgb16ToY, 0, 0, 0, 0xF800, 0x07E0, 0x001F, RY    , GY<<5, BY<<11, RGB2YUV_SHIFT+8)
1605 BGR2Y(uint16_t, rgb15ToY, 0, 0, 0, 0x7C00, 0x03E0, 0x001F, RY    , GY<<5, BY<<10, RGB2YUV_SHIFT+7)
1606
1607 #define BGR2UV(type, name, shr, shg, shb, maskr, maskg, maskb, RU, GU, BU, RV, GV, BV, S)\
1608 static inline void RENAME(name)(uint8_t *dstU, uint8_t *dstV, uint8_t *src, uint8_t *dummy, long width, uint32_t *unused)\
1609 {\
1610     int i;\
1611     for (i=0; i<width; i++)\
1612     {\
1613         int b= (((type*)src)[i]&maskb)>>shb;\
1614         int g= (((type*)src)[i]&maskg)>>shg;\
1615         int r= (((type*)src)[i]&maskr)>>shr;\
1616 \
1617         dstU[i]= ((RU)*r + (GU)*g + (BU)*b + (257<<((S)-1)))>>(S);\
1618         dstV[i]= ((RV)*r + (GV)*g + (BV)*b + (257<<((S)-1)))>>(S);\
1619     }\
1620 }\
1621 static inline void RENAME(name ## _half)(uint8_t *dstU, uint8_t *dstV, uint8_t *src, uint8_t *dummy, long width, uint32_t *unused)\
1622 {\
1623     int i;\
1624     for (i=0; i<width; i++)\
1625     {\
1626         int pix0= ((type*)src)[2*i+0];\
1627         int pix1= ((type*)src)[2*i+1];\
1628         int g= (pix0&maskg)+(pix1&maskg);\
1629         int b= ((pix0+pix1-g)&(maskb|(2*maskb)))>>shb;\
1630         int r= ((pix0+pix1-g)&(maskr|(2*maskr)))>>shr;\
1631 \
1632         g>>=shg;\
1633 \
1634         dstU[i]= ((RU)*r + (GU)*g + (BU)*b + (257<<(S)))>>((S)+1);\
1635         dstV[i]= ((RV)*r + (GV)*g + (BV)*b + (257<<(S)))>>((S)+1);\
1636     }\
1637 }
1638
1639 BGR2UV(uint32_t, bgr32ToUV,16, 0, 0, 0xFF0000, 0xFF00, 0x00FF, RU<< 8, GU   , BU<< 8, RV<< 8, GV   , BV<< 8, RGB2YUV_SHIFT+8)
1640 BGR2UV(uint32_t, rgb32ToUV, 0, 0,16, 0x00FF, 0xFF00, 0xFF0000, RU<< 8, GU   , BU<< 8, RV<< 8, GV   , BV<< 8, RGB2YUV_SHIFT+8)
1641 BGR2UV(uint16_t, bgr16ToUV, 0, 0, 0, 0x001F, 0x07E0, 0xF800, RU<<11, GU<<5, BU    , RV<<11, GV<<5, BV    , RGB2YUV_SHIFT+8)
1642 BGR2UV(uint16_t, bgr15ToUV, 0, 0, 0, 0x001F, 0x03E0, 0x7C00, RU<<10, GU<<5, BU    , RV<<10, GV<<5, BV    , RGB2YUV_SHIFT+7)
1643 BGR2UV(uint16_t, rgb16ToUV, 0, 0, 0, 0xF800, 0x07E0, 0x001F, RU    , GU<<5, BU<<11, RV    , GV<<5, BV<<11, RGB2YUV_SHIFT+8)
1644 BGR2UV(uint16_t, rgb15ToUV, 0, 0, 0, 0x7C00, 0x03E0, 0x001F, RU    , GU<<5, BU<<10, RV    , GV<<5, BV<<10, RGB2YUV_SHIFT+7)
1645
1646 #ifdef HAVE_MMX
1647 static inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, uint8_t *src, long width, int srcFormat)
1648 {
1649
1650     if(srcFormat == PIX_FMT_BGR24){
1651         __asm__ volatile(
1652             "movq  "MANGLE(ff_bgr24toY1Coeff)", %%mm5       \n\t"
1653             "movq  "MANGLE(ff_bgr24toY2Coeff)", %%mm6       \n\t"
1654             :
1655         );
1656     }else{
1657         __asm__ volatile(
1658             "movq  "MANGLE(ff_rgb24toY1Coeff)", %%mm5       \n\t"
1659             "movq  "MANGLE(ff_rgb24toY2Coeff)", %%mm6       \n\t"
1660             :
1661         );
1662     }
1663
1664     __asm__ volatile(
1665         "movq  "MANGLE(ff_bgr24toYOffset)", %%mm4   \n\t"
1666         "mov                        %2, %%"REG_a"   \n\t"
1667         "pxor                    %%mm7, %%mm7       \n\t"
1668         "1:                                         \n\t"
1669         PREFETCH"               64(%0)              \n\t"
1670         "movd                     (%0), %%mm0       \n\t"
1671         "movd                    2(%0), %%mm1       \n\t"
1672         "movd                    6(%0), %%mm2       \n\t"
1673         "movd                    8(%0), %%mm3       \n\t"
1674         "add                       $12, %0          \n\t"
1675         "punpcklbw               %%mm7, %%mm0       \n\t"
1676         "punpcklbw               %%mm7, %%mm1       \n\t"
1677         "punpcklbw               %%mm7, %%mm2       \n\t"
1678         "punpcklbw               %%mm7, %%mm3       \n\t"
1679         "pmaddwd                 %%mm5, %%mm0       \n\t"
1680         "pmaddwd                 %%mm6, %%mm1       \n\t"
1681         "pmaddwd                 %%mm5, %%mm2       \n\t"
1682         "pmaddwd                 %%mm6, %%mm3       \n\t"
1683         "paddd                   %%mm1, %%mm0       \n\t"
1684         "paddd                   %%mm3, %%mm2       \n\t"
1685         "paddd                   %%mm4, %%mm0       \n\t"
1686         "paddd                   %%mm4, %%mm2       \n\t"
1687         "psrad                     $15, %%mm0       \n\t"
1688         "psrad                     $15, %%mm2       \n\t"
1689         "packssdw                %%mm2, %%mm0       \n\t"
1690         "packuswb                %%mm0, %%mm0       \n\t"
1691         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1692         "add                        $4, %%"REG_a"   \n\t"
1693         " js                        1b              \n\t"
1694     : "+r" (src)
1695     : "r" (dst+width), "g" (-width)
1696     : "%"REG_a
1697     );
1698 }
1699
1700 static inline void RENAME(bgr24ToUV_mmx)(uint8_t *dstU, uint8_t *dstV, uint8_t *src, long width, int srcFormat)
1701 {
1702     __asm__ volatile(
1703         "movq                    24+%4, %%mm6       \n\t"
1704         "mov                        %3, %%"REG_a"   \n\t"
1705         "pxor                    %%mm7, %%mm7       \n\t"
1706         "1:                                         \n\t"
1707         PREFETCH"               64(%0)              \n\t"
1708         "movd                     (%0), %%mm0       \n\t"
1709         "movd                    2(%0), %%mm1       \n\t"
1710         "punpcklbw               %%mm7, %%mm0       \n\t"
1711         "punpcklbw               %%mm7, %%mm1       \n\t"
1712         "movq                    %%mm0, %%mm2       \n\t"
1713         "movq                    %%mm1, %%mm3       \n\t"
1714         "pmaddwd                    %4, %%mm0       \n\t"
1715         "pmaddwd                  8+%4, %%mm1       \n\t"
1716         "pmaddwd                 16+%4, %%mm2       \n\t"
1717         "pmaddwd                 %%mm6, %%mm3       \n\t"
1718         "paddd                   %%mm1, %%mm0       \n\t"
1719         "paddd                   %%mm3, %%mm2       \n\t"
1720
1721         "movd                    6(%0), %%mm1       \n\t"
1722         "movd                    8(%0), %%mm3       \n\t"
1723         "add                       $12, %0          \n\t"
1724         "punpcklbw               %%mm7, %%mm1       \n\t"
1725         "punpcklbw               %%mm7, %%mm3       \n\t"
1726         "movq                    %%mm1, %%mm4       \n\t"
1727         "movq                    %%mm3, %%mm5       \n\t"
1728         "pmaddwd                    %4, %%mm1       \n\t"
1729         "pmaddwd                  8+%4, %%mm3       \n\t"
1730         "pmaddwd                 16+%4, %%mm4       \n\t"
1731         "pmaddwd                 %%mm6, %%mm5       \n\t"
1732         "paddd                   %%mm3, %%mm1       \n\t"
1733         "paddd                   %%mm5, %%mm4       \n\t"
1734
1735         "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3       \n\t"
1736         "paddd                   %%mm3, %%mm0       \n\t"
1737         "paddd                   %%mm3, %%mm2       \n\t"
1738         "paddd                   %%mm3, %%mm1       \n\t"
1739         "paddd                   %%mm3, %%mm4       \n\t"
1740         "psrad                     $15, %%mm0       \n\t"
1741         "psrad                     $15, %%mm2       \n\t"
1742         "psrad                     $15, %%mm1       \n\t"
1743         "psrad                     $15, %%mm4       \n\t"
1744         "packssdw                %%mm1, %%mm0       \n\t"
1745         "packssdw                %%mm4, %%mm2       \n\t"
1746         "packuswb                %%mm0, %%mm0       \n\t"
1747         "packuswb                %%mm2, %%mm2       \n\t"
1748         "movd                %%mm0, (%1, %%"REG_a") \n\t"
1749         "movd                %%mm2, (%2, %%"REG_a") \n\t"
1750         "add                        $4, %%"REG_a"   \n\t"
1751         " js                        1b              \n\t"
1752     : "+r" (src)
1753     : "r" (dstU+width), "r" (dstV+width), "g" (-width), "m"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24][0])
1754     : "%"REG_a
1755     );
1756 }
1757 #endif
1758
1759 static inline void RENAME(bgr24ToY)(uint8_t *dst, uint8_t *src, long width, uint32_t *unused)
1760 {
1761 #ifdef HAVE_MMX
1762     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1763 #else
1764     int i;
1765     for (i=0; i<width; i++)
1766     {
1767         int b= src[i*3+0];
1768         int g= src[i*3+1];
1769         int r= src[i*3+2];
1770
1771         dst[i]= ((RY*r + GY*g + BY*b + (33<<(RGB2YUV_SHIFT-1)))>>RGB2YUV_SHIFT);
1772     }
1773 #endif /* HAVE_MMX */
1774 }
1775
1776 static inline void RENAME(bgr24ToUV)(uint8_t *dstU, uint8_t *dstV, uint8_t *src1, uint8_t *src2, long width, uint32_t *unused)
1777 {
1778 #ifdef HAVE_MMX
1779     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1780 #else
1781     int i;
1782     for (i=0; i<width; i++)
1783     {
1784         int b= src1[3*i + 0];
1785         int g= src1[3*i + 1];
1786         int r= src1[3*i + 2];
1787
1788         dstU[i]= (RU*r + GU*g + BU*b + (257<<(RGB2YUV_SHIFT-1)))>>RGB2YUV_SHIFT;
1789         dstV[i]= (RV*r + GV*g + BV*b + (257<<(RGB2YUV_SHIFT-1)))>>RGB2YUV_SHIFT;
1790     }
1791 #endif /* HAVE_MMX */
1792     assert(src1 == src2);
1793 }
1794
1795 static inline void RENAME(bgr24ToUV_half)(uint8_t *dstU, uint8_t *dstV, uint8_t *src1, uint8_t *src2, long width, uint32_t *unused)
1796 {
1797     int i;
1798     for (i=0; i<width; i++)
1799     {
1800         int b= src1[6*i + 0] + src1[6*i + 3];
1801         int g= src1[6*i + 1] + src1[6*i + 4];
1802         int r= src1[6*i + 2] + src1[6*i + 5];
1803
1804         dstU[i]= (RU*r + GU*g + BU*b + (257<<RGB2YUV_SHIFT))>>(RGB2YUV_SHIFT+1);
1805         dstV[i]= (RV*r + GV*g + BV*b + (257<<RGB2YUV_SHIFT))>>(RGB2YUV_SHIFT+1);
1806     }
1807     assert(src1 == src2);
1808 }
1809
1810 static inline void RENAME(rgb24ToY)(uint8_t *dst, uint8_t *src, long width, uint32_t *unused)
1811 {
1812 #ifdef HAVE_MMX
1813     RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1814 #else
1815     int i;
1816     for (i=0; i<width; i++)
1817     {
1818         int r= src[i*3+0];
1819         int g= src[i*3+1];
1820         int b= src[i*3+2];
1821
1822         dst[i]= ((RY*r + GY*g + BY*b + (33<<(RGB2YUV_SHIFT-1)))>>RGB2YUV_SHIFT);
1823     }
1824 #endif
1825 }
1826
1827 static inline void RENAME(rgb24ToUV)(uint8_t *dstU, uint8_t *dstV, uint8_t *src1, uint8_t *src2, long width, uint32_t *unused)
1828 {
1829     int i;
1830     assert(src1==src2);
1831 #ifdef HAVE_MMX
1832     RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1833 #else
1834     for (i=0; i<width; i++)
1835     {
1836         int r= src1[3*i + 0];
1837         int g= src1[3*i + 1];
1838         int b= src1[3*i + 2];
1839
1840         dstU[i]= (RU*r + GU*g + BU*b + (257<<(RGB2YUV_SHIFT-1)))>>RGB2YUV_SHIFT;
1841         dstV[i]= (RV*r + GV*g + BV*b + (257<<(RGB2YUV_SHIFT-1)))>>RGB2YUV_SHIFT;
1842     }
1843 #endif
1844 }
1845
1846 static inline void RENAME(rgb24ToUV_half)(uint8_t *dstU, uint8_t *dstV, uint8_t *src1, uint8_t *src2, long width, uint32_t *unused)
1847 {
1848     int i;
1849     assert(src1==src2);
1850     for (i=0; i<width; i++)
1851     {
1852         int r= src1[6*i + 0] + src1[6*i + 3];
1853         int g= src1[6*i + 1] + src1[6*i + 4];
1854         int b= src1[6*i + 2] + src1[6*i + 5];
1855
1856         dstU[i]= (RU*r + GU*g + BU*b + (257<<RGB2YUV_SHIFT))>>(RGB2YUV_SHIFT+1);
1857         dstV[i]= (RV*r + GV*g + BV*b + (257<<RGB2YUV_SHIFT))>>(RGB2YUV_SHIFT+1);
1858     }
1859 }
1860
1861
1862 static inline void RENAME(palToY)(uint8_t *dst, uint8_t *src, long width, uint32_t *pal)
1863 {
1864     int i;
1865     for (i=0; i<width; i++)
1866     {
1867         int d= src[i];
1868
1869         dst[i]= pal[d] & 0xFF;
1870     }
1871 }
1872
1873 static inline void RENAME(palToUV)(uint8_t *dstU, uint8_t *dstV, uint8_t *src1, uint8_t *src2, long width, uint32_t *pal)
1874 {
1875     int i;
1876     assert(src1 == src2);
1877     for (i=0; i<width; i++)
1878     {
1879         int p= pal[src1[i]];
1880
1881         dstU[i]= p>>8;
1882         dstV[i]= p>>16;
1883     }
1884 }
1885
1886 static inline void RENAME(monowhite2Y)(uint8_t *dst, uint8_t *src, long width, uint32_t *unused)
1887 {
1888     int i, j;
1889     for (i=0; i<width/8; i++){
1890         int d= ~src[i];
1891         for(j=0; j<8; j++)
1892             dst[8*i+j]= ((d>>(7-j))&1)*255;
1893     }
1894 }
1895
1896 static inline void RENAME(monoblack2Y)(uint8_t *dst, uint8_t *src, long width, uint32_t *unused)
1897 {
1898     int i, j;
1899     for (i=0; i<width/8; i++){
1900         int d= src[i];
1901         for(j=0; j<8; j++)
1902             dst[8*i+j]= ((d>>(7-j))&1)*255;
1903     }
1904 }
1905
1906 // bilinear / bicubic scaling
1907 static inline void RENAME(hScale)(int16_t *dst, int dstW, uint8_t *src, int srcW, int xInc,
1908                                   int16_t *filter, int16_t *filterPos, long filterSize)
1909 {
1910 #ifdef HAVE_MMX
1911     assert(filterSize % 4 == 0 && filterSize>0);
1912     if (filterSize==4) // Always true for upscaling, sometimes for down, too.
1913     {
1914         long counter= -2*dstW;
1915         filter-= counter*2;
1916         filterPos-= counter/2;
1917         dst-= counter/2;
1918         __asm__ volatile(
1919 #if defined(PIC)
1920         "push            %%"REG_b"              \n\t"
1921 #endif
1922         "pxor                %%mm7, %%mm7       \n\t"
1923         "push           %%"REG_BP"              \n\t" // we use 7 regs here ...
1924         "mov             %%"REG_a", %%"REG_BP"  \n\t"
1925         ASMALIGN(4)
1926         "1:                                     \n\t"
1927         "movzwl   (%2, %%"REG_BP"), %%eax       \n\t"
1928         "movzwl  2(%2, %%"REG_BP"), %%ebx       \n\t"
1929         "movq  (%1, %%"REG_BP", 4), %%mm1       \n\t"
1930         "movq 8(%1, %%"REG_BP", 4), %%mm3       \n\t"
1931         "movd      (%3, %%"REG_a"), %%mm0       \n\t"
1932         "movd      (%3, %%"REG_b"), %%mm2       \n\t"
1933         "punpcklbw           %%mm7, %%mm0       \n\t"
1934         "punpcklbw           %%mm7, %%mm2       \n\t"
1935         "pmaddwd             %%mm1, %%mm0       \n\t"
1936         "pmaddwd             %%mm2, %%mm3       \n\t"
1937         "movq                %%mm0, %%mm4       \n\t"
1938         "punpckldq           %%mm3, %%mm0       \n\t"
1939         "punpckhdq           %%mm3, %%mm4       \n\t"
1940         "paddd               %%mm4, %%mm0       \n\t"
1941         "psrad                  $7, %%mm0       \n\t"
1942         "packssdw            %%mm0, %%mm0       \n\t"
1943         "movd                %%mm0, (%4, %%"REG_BP")    \n\t"
1944         "add                    $4, %%"REG_BP"  \n\t"
1945         " jnc                   1b              \n\t"
1946
1947         "pop            %%"REG_BP"              \n\t"
1948 #if defined(PIC)
1949         "pop             %%"REG_b"              \n\t"
1950 #endif
1951         : "+a" (counter)
1952         : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1953 #if !defined(PIC)
1954         : "%"REG_b
1955 #endif
1956         );
1957     }
1958     else if (filterSize==8)
1959     {
1960         long counter= -2*dstW;
1961         filter-= counter*4;
1962         filterPos-= counter/2;
1963         dst-= counter/2;
1964         __asm__ volatile(
1965 #if defined(PIC)
1966         "push             %%"REG_b"             \n\t"
1967 #endif
1968         "pxor                 %%mm7, %%mm7      \n\t"
1969         "push            %%"REG_BP"             \n\t" // we use 7 regs here ...
1970         "mov              %%"REG_a", %%"REG_BP" \n\t"
1971         ASMALIGN(4)
1972         "1:                                     \n\t"
1973         "movzwl    (%2, %%"REG_BP"), %%eax      \n\t"
1974         "movzwl   2(%2, %%"REG_BP"), %%ebx      \n\t"
1975         "movq   (%1, %%"REG_BP", 8), %%mm1      \n\t"
1976         "movq 16(%1, %%"REG_BP", 8), %%mm3      \n\t"
1977         "movd       (%3, %%"REG_a"), %%mm0      \n\t"
1978         "movd       (%3, %%"REG_b"), %%mm2      \n\t"
1979         "punpcklbw            %%mm7, %%mm0      \n\t"
1980         "punpcklbw            %%mm7, %%mm2      \n\t"
1981         "pmaddwd              %%mm1, %%mm0      \n\t"
1982         "pmaddwd              %%mm2, %%mm3      \n\t"
1983
1984         "movq  8(%1, %%"REG_BP", 8), %%mm1      \n\t"
1985         "movq 24(%1, %%"REG_BP", 8), %%mm5      \n\t"
1986         "movd      4(%3, %%"REG_a"), %%mm4      \n\t"
1987         "movd      4(%3, %%"REG_b"), %%mm2      \n\t"
1988         "punpcklbw            %%mm7, %%mm4      \n\t"
1989         "punpcklbw            %%mm7, %%mm2      \n\t"
1990         "pmaddwd              %%mm1, %%mm4      \n\t"
1991         "pmaddwd              %%mm2, %%mm5      \n\t"
1992         "paddd                %%mm4, %%mm0      \n\t"
1993         "paddd                %%mm5, %%mm3      \n\t"
1994         "movq                 %%mm0, %%mm4      \n\t"
1995         "punpckldq            %%mm3, %%mm0      \n\t"
1996         "punpckhdq            %%mm3, %%mm4      \n\t"
1997         "paddd                %%mm4, %%mm0      \n\t"
1998         "psrad                   $7, %%mm0      \n\t"
1999         "packssdw             %%mm0, %%mm0      \n\t"
2000         "movd                 %%mm0, (%4, %%"REG_BP")   \n\t"
2001         "add                     $4, %%"REG_BP" \n\t"
2002         " jnc                    1b             \n\t"
2003
2004         "pop             %%"REG_BP"             \n\t"
2005 #if defined(PIC)
2006         "pop              %%"REG_b"             \n\t"
2007 #endif
2008         : "+a" (counter)
2009         : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
2010 #if !defined(PIC)
2011         : "%"REG_b
2012 #endif
2013         );
2014     }
2015     else
2016     {
2017         uint8_t *offset = src+filterSize;
2018         long counter= -2*dstW;
2019         //filter-= counter*filterSize/2;
2020         filterPos-= counter/2;
2021         dst-= counter/2;
2022         __asm__ volatile(
2023         "pxor                  %%mm7, %%mm7     \n\t"
2024         ASMALIGN(4)
2025         "1:                                     \n\t"
2026         "mov                      %2, %%"REG_c" \n\t"
2027         "movzwl      (%%"REG_c", %0), %%eax     \n\t"
2028         "movzwl     2(%%"REG_c", %0), %%edx     \n\t"
2029         "mov                      %5, %%"REG_c" \n\t"
2030         "pxor                  %%mm4, %%mm4     \n\t"
2031         "pxor                  %%mm5, %%mm5     \n\t"
2032         "2:                                     \n\t"
2033         "movq                   (%1), %%mm1     \n\t"
2034         "movq               (%1, %6), %%mm3     \n\t"
2035         "movd (%%"REG_c", %%"REG_a"), %%mm0     \n\t"
2036         "movd (%%"REG_c", %%"REG_d"), %%mm2     \n\t"
2037         "punpcklbw             %%mm7, %%mm0     \n\t"
2038         "punpcklbw             %%mm7, %%mm2     \n\t"
2039         "pmaddwd               %%mm1, %%mm0     \n\t"
2040         "pmaddwd               %%mm2, %%mm3     \n\t"
2041         "paddd                 %%mm3, %%mm5     \n\t"
2042         "paddd                 %%mm0, %%mm4     \n\t"
2043         "add                      $8, %1        \n\t"
2044         "add                      $4, %%"REG_c" \n\t"
2045         "cmp                      %4, %%"REG_c" \n\t"
2046         " jb                      2b            \n\t"
2047         "add                      %6, %1        \n\t"
2048         "movq                  %%mm4, %%mm0     \n\t"
2049         "punpckldq             %%mm5, %%mm4     \n\t"
2050         "punpckhdq             %%mm5, %%mm0     \n\t"
2051         "paddd                 %%mm0, %%mm4     \n\t"
2052         "psrad                    $7, %%mm4     \n\t"
2053         "packssdw              %%mm4, %%mm4     \n\t"
2054         "mov                      %3, %%"REG_a" \n\t"
2055         "movd                  %%mm4, (%%"REG_a", %0)   \n\t"
2056         "add                      $4, %0        \n\t"
2057         " jnc                     1b            \n\t"
2058
2059         : "+r" (counter), "+r" (filter)
2060         : "m" (filterPos), "m" (dst), "m"(offset),
2061           "m" (src), "r" (filterSize*2)
2062         : "%"REG_a, "%"REG_c, "%"REG_d
2063         );
2064     }
2065 #else
2066 #ifdef HAVE_ALTIVEC
2067     hScale_altivec_real(dst, dstW, src, srcW, xInc, filter, filterPos, filterSize);
2068 #else
2069     int i;
2070     for (i=0; i<dstW; i++)
2071     {
2072         int j;
2073         int srcPos= filterPos[i];
2074         int val=0;
2075         //printf("filterPos: %d\n", filterPos[i]);
2076         for (j=0; j<filterSize; j++)
2077         {
2078             //printf("filter: %d, src: %d\n", filter[i], src[srcPos + j]);
2079             val += ((int)src[srcPos + j])*filter[filterSize*i + j];
2080         }
2081         //filter += hFilterSize;
2082         dst[i] = FFMIN(val>>7, (1<<15)-1); // the cubic equation does overflow ...
2083         //dst[i] = val>>7;
2084     }
2085 #endif /* HAVE_ALTIVEC */
2086 #endif /* HAVE_MMX */
2087 }
2088       // *** horizontal scale Y line to temp buffer
2089 static inline void RENAME(hyscale)(SwsContext *c, uint16_t *dst, long dstWidth, uint8_t *src, int srcW, int xInc,
2090                                    int flags, int canMMX2BeUsed, int16_t *hLumFilter,
2091                                    int16_t *hLumFilterPos, int hLumFilterSize, void *funnyYCode,
2092                                    int srcFormat, uint8_t *formatConvBuffer, int16_t *mmx2Filter,
2093                                    int32_t *mmx2FilterPos, uint32_t *pal)
2094 {
2095     if (srcFormat==PIX_FMT_YUYV422 || srcFormat==PIX_FMT_GRAY16BE)
2096     {
2097         RENAME(yuy2ToY)(formatConvBuffer, src, srcW, pal);
2098         src= formatConvBuffer;
2099     }
2100     else if (srcFormat==PIX_FMT_UYVY422 || srcFormat==PIX_FMT_GRAY16LE)
2101     {
2102         RENAME(uyvyToY)(formatConvBuffer, src, srcW, pal);
2103         src= formatConvBuffer;
2104     }
2105     else if (srcFormat==PIX_FMT_RGB32)
2106     {
2107         RENAME(bgr32ToY)(formatConvBuffer, src, srcW, pal);
2108         src= formatConvBuffer;
2109     }
2110     else if (srcFormat==PIX_FMT_RGB32_1)
2111     {
2112         RENAME(bgr32ToY)(formatConvBuffer, src+ALT32_CORR, srcW, pal);
2113         src= formatConvBuffer;
2114     }
2115     else if (srcFormat==PIX_FMT_BGR24)
2116     {
2117         RENAME(bgr24ToY)(formatConvBuffer, src, srcW, pal);
2118         src= formatConvBuffer;
2119     }
2120     else if (srcFormat==PIX_FMT_BGR565)
2121     {
2122         RENAME(bgr16ToY)(formatConvBuffer, src, srcW, pal);
2123         src= formatConvBuffer;
2124     }
2125     else if (srcFormat==PIX_FMT_BGR555)
2126     {
2127         RENAME(bgr15ToY)(formatConvBuffer, src, srcW, pal);
2128         src= formatConvBuffer;
2129     }
2130     else if (srcFormat==PIX_FMT_BGR32)
2131     {
2132         RENAME(rgb32ToY)(formatConvBuffer, src, srcW, pal);
2133         src= formatConvBuffer;
2134     }
2135     else if (srcFormat==PIX_FMT_BGR32_1)
2136     {
2137         RENAME(rgb32ToY)(formatConvBuffer, src+ALT32_CORR, srcW, pal);
2138         src= formatConvBuffer;
2139     }
2140     else if (srcFormat==PIX_FMT_RGB24)
2141     {
2142         RENAME(rgb24ToY)(formatConvBuffer, src, srcW, pal);
2143         src= formatConvBuffer;
2144     }
2145     else if (srcFormat==PIX_FMT_RGB565)
2146     {
2147         RENAME(rgb16ToY)(formatConvBuffer, src, srcW, pal);
2148         src= formatConvBuffer;
2149     }
2150     else if (srcFormat==PIX_FMT_RGB555)
2151     {
2152         RENAME(rgb15ToY)(formatConvBuffer, src, srcW, pal);
2153         src= formatConvBuffer;
2154     }
2155     else if (srcFormat==PIX_FMT_RGB8 || srcFormat==PIX_FMT_BGR8 || srcFormat==PIX_FMT_PAL8 || srcFormat==PIX_FMT_BGR4_BYTE  || srcFormat==PIX_FMT_RGB4_BYTE)
2156     {
2157         RENAME(palToY)(formatConvBuffer, src, srcW, pal);
2158         src= formatConvBuffer;
2159     }
2160     else if (srcFormat==PIX_FMT_MONOBLACK)
2161     {
2162         RENAME(monoblack2Y)(formatConvBuffer, src, srcW, pal);
2163         src= formatConvBuffer;
2164     }
2165     else if (srcFormat==PIX_FMT_MONOWHITE)
2166     {
2167         RENAME(monowhite2Y)(formatConvBuffer, src, srcW, pal);
2168         src= formatConvBuffer;
2169     }
2170
2171 #ifdef HAVE_MMX
2172     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2173     if (!(flags&SWS_FAST_BILINEAR) || (!canMMX2BeUsed))
2174 #else
2175     if (!(flags&SWS_FAST_BILINEAR))
2176 #endif
2177     {
2178         RENAME(hScale)(dst, dstWidth, src, srcW, xInc, hLumFilter, hLumFilterPos, hLumFilterSize);
2179     }
2180     else // fast bilinear upscale / crap downscale
2181     {
2182 #if defined(ARCH_X86)
2183 #ifdef HAVE_MMX2
2184         int i;
2185 #if defined(PIC)
2186         uint64_t ebxsave __attribute__((aligned(8)));
2187 #endif
2188         if (canMMX2BeUsed)
2189         {
2190             __asm__ volatile(
2191 #if defined(PIC)
2192             "mov               %%"REG_b", %5        \n\t"
2193 #endif
2194             "pxor                  %%mm7, %%mm7     \n\t"
2195             "mov                      %0, %%"REG_c" \n\t"
2196             "mov                      %1, %%"REG_D" \n\t"
2197             "mov                      %2, %%"REG_d" \n\t"
2198             "mov                      %3, %%"REG_b" \n\t"
2199             "xor               %%"REG_a", %%"REG_a" \n\t" // i
2200             PREFETCH"        (%%"REG_c")            \n\t"
2201             PREFETCH"      32(%%"REG_c")            \n\t"
2202             PREFETCH"      64(%%"REG_c")            \n\t"
2203
2204 #ifdef ARCH_X86_64
2205
2206 #define FUNNY_Y_CODE \
2207             "movl            (%%"REG_b"), %%esi     \n\t"\
2208             "call                    *%4            \n\t"\
2209             "movl (%%"REG_b", %%"REG_a"), %%esi     \n\t"\
2210             "add               %%"REG_S", %%"REG_c" \n\t"\
2211             "add               %%"REG_a", %%"REG_D" \n\t"\
2212             "xor               %%"REG_a", %%"REG_a" \n\t"\
2213
2214 #else
2215
2216 #define FUNNY_Y_CODE \
2217             "movl (%%"REG_b"), %%esi        \n\t"\
2218             "call         *%4                       \n\t"\
2219             "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
2220             "add               %%"REG_a", %%"REG_D" \n\t"\
2221             "xor               %%"REG_a", %%"REG_a" \n\t"\
2222
2223 #endif /* ARCH_X86_64 */
2224
2225 FUNNY_Y_CODE
2226 FUNNY_Y_CODE
2227 FUNNY_Y_CODE
2228 FUNNY_Y_CODE
2229 FUNNY_Y_CODE
2230 FUNNY_Y_CODE
2231 FUNNY_Y_CODE
2232 FUNNY_Y_CODE
2233
2234 #if defined(PIC)
2235             "mov                      %5, %%"REG_b" \n\t"
2236 #endif
2237             :: "m" (src), "m" (dst), "m" (mmx2Filter), "m" (mmx2FilterPos),
2238             "m" (funnyYCode)
2239 #if defined(PIC)
2240             ,"m" (ebxsave)
2241 #endif
2242             : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2243 #if !defined(PIC)
2244             ,"%"REG_b
2245 #endif
2246             );
2247             for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) dst[i] = src[srcW-1]*128;
2248         }
2249         else
2250         {
2251 #endif /* HAVE_MMX2 */
2252         long xInc_shr16 = xInc >> 16;
2253         uint16_t xInc_mask = xInc & 0xffff;
2254         //NO MMX just normal asm ...
2255         __asm__ volatile(
2256         "xor %%"REG_a", %%"REG_a"            \n\t" // i
2257         "xor %%"REG_d", %%"REG_d"            \n\t" // xx
2258         "xorl    %%ecx, %%ecx                \n\t" // 2*xalpha
2259         ASMALIGN(4)
2260         "1:                                  \n\t"
2261         "movzbl    (%0, %%"REG_d"), %%edi    \n\t" //src[xx]
2262         "movzbl   1(%0, %%"REG_d"), %%esi    \n\t" //src[xx+1]
2263         "subl    %%edi, %%esi                \n\t" //src[xx+1] - src[xx]
2264         "imull   %%ecx, %%esi                \n\t" //(src[xx+1] - src[xx])*2*xalpha
2265         "shll      $16, %%edi                \n\t"
2266         "addl    %%edi, %%esi                \n\t" //src[xx+1]*2*xalpha + src[xx]*(1-2*xalpha)
2267         "mov        %1, %%"REG_D"            \n\t"
2268         "shrl       $9, %%esi                \n\t"
2269         "movw     %%si, (%%"REG_D", %%"REG_a", 2)   \n\t"
2270         "addw       %4, %%cx                 \n\t" //2*xalpha += xInc&0xFF
2271         "adc        %3, %%"REG_d"            \n\t" //xx+= xInc>>8 + carry
2272
2273         "movzbl    (%0, %%"REG_d"), %%edi    \n\t" //src[xx]
2274         "movzbl   1(%0, %%"REG_d"), %%esi    \n\t" //src[xx+1]
2275         "subl    %%edi, %%esi                \n\t" //src[xx+1] - src[xx]
2276         "imull   %%ecx, %%esi                \n\t" //(src[xx+1] - src[xx])*2*xalpha
2277         "shll      $16, %%edi                \n\t"
2278         "addl    %%edi, %%esi                \n\t" //src[xx+1]*2*xalpha + src[xx]*(1-2*xalpha)
2279         "mov        %1, %%"REG_D"            \n\t"
2280         "shrl       $9, %%esi                \n\t"
2281         "movw     %%si, 2(%%"REG_D", %%"REG_a", 2)  \n\t"
2282         "addw       %4, %%cx                 \n\t" //2*xalpha += xInc&0xFF
2283         "adc        %3, %%"REG_d"            \n\t" //xx+= xInc>>8 + carry
2284
2285
2286         "add        $2, %%"REG_a"            \n\t"
2287         "cmp        %2, %%"REG_a"            \n\t"
2288         " jb        1b                       \n\t"
2289
2290
2291         :: "r" (src), "m" (dst), "m" (dstWidth), "m" (xInc_shr16), "m" (xInc_mask)
2292         : "%"REG_a, "%"REG_d, "%ecx", "%"REG_D, "%esi"
2293         );
2294 #ifdef HAVE_MMX2
2295         } //if MMX2 can't be used
2296 #endif
2297 #else
2298         int i;
2299         unsigned int xpos=0;
2300         for (i=0;i<dstWidth;i++)
2301         {
2302             register unsigned int xx=xpos>>16;
2303             register unsigned int xalpha=(xpos&0xFFFF)>>9;
2304             dst[i]= (src[xx]<<7) + (src[xx+1] - src[xx])*xalpha;
2305             xpos+=xInc;
2306         }
2307 #endif /* defined(ARCH_X86) */
2308     }
2309
2310     if(c->srcRange != c->dstRange && !(isRGB(c->dstFormat) || isBGR(c->dstFormat))){
2311         int i;
2312         //FIXME all pal and rgb srcFormats could do this convertion as well
2313         //FIXME all scalers more complex than bilinear could do half of this transform
2314         if(c->srcRange){
2315             for (i=0; i<dstWidth; i++)
2316                 dst[i]= (dst[i]*14071 + 33561947)>>14;
2317         }else{
2318             for (i=0; i<dstWidth; i++)
2319                 dst[i]= (FFMIN(dst[i],30189)*19077 - 39057361)>>14;
2320         }
2321     }
2322 }
2323
2324 inline static void RENAME(hcscale)(SwsContext *c, uint16_t *dst, long dstWidth, uint8_t *src1, uint8_t *src2,
2325                                    int srcW, int xInc, int flags, int canMMX2BeUsed, int16_t *hChrFilter,
2326                                    int16_t *hChrFilterPos, int hChrFilterSize, void *funnyUVCode,
2327                                    int srcFormat, uint8_t *formatConvBuffer, int16_t *mmx2Filter,
2328                                    int32_t *mmx2FilterPos, uint32_t *pal)
2329 {
2330     if (srcFormat==PIX_FMT_YUYV422)
2331     {
2332         RENAME(yuy2ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2333         src1= formatConvBuffer;
2334         src2= formatConvBuffer+VOFW;
2335     }
2336     else if (srcFormat==PIX_FMT_UYVY422)
2337     {
2338         RENAME(uyvyToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2339         src1= formatConvBuffer;
2340         src2= formatConvBuffer+VOFW;
2341     }
2342     else if (srcFormat==PIX_FMT_RGB32)
2343     {
2344         if(c->chrSrcHSubSample)
2345             RENAME(bgr32ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2346         else
2347             RENAME(bgr32ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2348         src1= formatConvBuffer;
2349         src2= formatConvBuffer+VOFW;
2350     }
2351     else if (srcFormat==PIX_FMT_RGB32_1)
2352     {
2353         if(c->chrSrcHSubSample)
2354             RENAME(bgr32ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1+ALT32_CORR, src2+ALT32_CORR, srcW, pal);
2355         else
2356             RENAME(bgr32ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1+ALT32_CORR, src2+ALT32_CORR, srcW, pal);
2357         src1= formatConvBuffer;
2358         src2= formatConvBuffer+VOFW;
2359     }
2360     else if (srcFormat==PIX_FMT_BGR24)
2361     {
2362         if(c->chrSrcHSubSample)
2363             RENAME(bgr24ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2364         else
2365             RENAME(bgr24ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2366         src1= formatConvBuffer;
2367         src2= formatConvBuffer+VOFW;
2368     }
2369     else if (srcFormat==PIX_FMT_BGR565)
2370     {
2371         if(c->chrSrcHSubSample)
2372             RENAME(bgr16ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2373         else
2374             RENAME(bgr16ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2375         src1= formatConvBuffer;
2376         src2= formatConvBuffer+VOFW;
2377     }
2378     else if (srcFormat==PIX_FMT_BGR555)
2379     {
2380         if(c->chrSrcHSubSample)
2381             RENAME(bgr15ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2382         else
2383             RENAME(bgr15ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2384         src1= formatConvBuffer;
2385         src2= formatConvBuffer+VOFW;
2386     }
2387     else if (srcFormat==PIX_FMT_BGR32)
2388     {
2389         if(c->chrSrcHSubSample)
2390             RENAME(rgb32ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2391         else
2392             RENAME(rgb32ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2393         src1= formatConvBuffer;
2394         src2= formatConvBuffer+VOFW;
2395     }
2396     else if (srcFormat==PIX_FMT_BGR32_1)
2397     {
2398         if(c->chrSrcHSubSample)
2399             RENAME(rgb32ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1+ALT32_CORR, src2+ALT32_CORR, srcW, pal);
2400         else
2401             RENAME(rgb32ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1+ALT32_CORR, src2+ALT32_CORR, srcW, pal);
2402         src1= formatConvBuffer;
2403         src2= formatConvBuffer+VOFW;
2404     }
2405     else if (srcFormat==PIX_FMT_RGB24)
2406     {
2407         if(c->chrSrcHSubSample)
2408             RENAME(rgb24ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2409         else
2410             RENAME(rgb24ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2411         src1= formatConvBuffer;
2412         src2= formatConvBuffer+VOFW;
2413     }
2414     else if (srcFormat==PIX_FMT_RGB565)
2415     {
2416         if(c->chrSrcHSubSample)
2417             RENAME(rgb16ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2418         else
2419             RENAME(rgb16ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2420         src1= formatConvBuffer;
2421         src2= formatConvBuffer+VOFW;
2422     }
2423     else if (srcFormat==PIX_FMT_RGB555)
2424     {
2425         if(c->chrSrcHSubSample)
2426             RENAME(rgb15ToUV_half)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2427         else
2428             RENAME(rgb15ToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2429         src1= formatConvBuffer;
2430         src2= formatConvBuffer+VOFW;
2431     }
2432     else if (isGray(srcFormat) || srcFormat==PIX_FMT_MONOBLACK || srcFormat==PIX_FMT_MONOWHITE)
2433     {
2434         return;
2435     }
2436     else if (srcFormat==PIX_FMT_RGB8 || srcFormat==PIX_FMT_BGR8 || srcFormat==PIX_FMT_PAL8 || srcFormat==PIX_FMT_BGR4_BYTE  || srcFormat==PIX_FMT_RGB4_BYTE)
2437     {
2438         RENAME(palToUV)(formatConvBuffer, formatConvBuffer+VOFW, src1, src2, srcW, pal);
2439         src1= formatConvBuffer;
2440         src2= formatConvBuffer+VOFW;
2441     }
2442
2443 #ifdef HAVE_MMX
2444     // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2445     if (!(flags&SWS_FAST_BILINEAR) || (!canMMX2BeUsed))
2446 #else
2447     if (!(flags&SWS_FAST_BILINEAR))
2448 #endif
2449     {
2450         RENAME(hScale)(dst     , dstWidth, src1, srcW, xInc, hChrFilter, hChrFilterPos, hChrFilterSize);
2451         RENAME(hScale)(dst+VOFW, dstWidth, src2, srcW, xInc, hChrFilter, hChrFilterPos, hChrFilterSize);
2452     }
2453     else // fast bilinear upscale / crap downscale
2454     {
2455 #if defined(ARCH_X86)
2456 #ifdef HAVE_MMX2
2457         int i;
2458 #if defined(PIC)
2459         uint64_t ebxsave __attribute__((aligned(8)));
2460 #endif
2461         if (canMMX2BeUsed)
2462         {
2463             __asm__ volatile(
2464 #if defined(PIC)
2465             "mov          %%"REG_b", %6         \n\t"
2466 #endif
2467             "pxor             %%mm7, %%mm7      \n\t"
2468             "mov                 %0, %%"REG_c"  \n\t"
2469             "mov                 %1, %%"REG_D"  \n\t"
2470             "mov                 %2, %%"REG_d"  \n\t"
2471             "mov                 %3, %%"REG_b"  \n\t"
2472             "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2473             PREFETCH"   (%%"REG_c")             \n\t"
2474             PREFETCH" 32(%%"REG_c")             \n\t"
2475             PREFETCH" 64(%%"REG_c")             \n\t"
2476
2477 #ifdef ARCH_X86_64
2478
2479 #define FUNNY_UV_CODE \
2480             "movl       (%%"REG_b"), %%esi      \n\t"\
2481             "call               *%4             \n\t"\
2482             "movl (%%"REG_b", %%"REG_a"), %%esi \n\t"\
2483             "add          %%"REG_S", %%"REG_c"  \n\t"\
2484             "add          %%"REG_a", %%"REG_D"  \n\t"\
2485             "xor          %%"REG_a", %%"REG_a"  \n\t"\
2486
2487 #else
2488
2489 #define FUNNY_UV_CODE \
2490             "movl       (%%"REG_b"), %%esi      \n\t"\
2491             "call               *%4             \n\t"\
2492             "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
2493             "add          %%"REG_a", %%"REG_D"  \n\t"\
2494             "xor          %%"REG_a", %%"REG_a"  \n\t"\
2495
2496 #endif /* ARCH_X86_64 */
2497
2498 FUNNY_UV_CODE
2499 FUNNY_UV_CODE
2500 FUNNY_UV_CODE
2501 FUNNY_UV_CODE
2502             "xor          %%"REG_a", %%"REG_a"  \n\t" // i
2503             "mov                 %5, %%"REG_c"  \n\t" // src
2504             "mov                 %1, %%"REG_D"  \n\t" // buf1
2505             "add              $"AV_STRINGIFY(VOF)", %%"REG_D"  \n\t"
2506             PREFETCH"   (%%"REG_c")             \n\t"
2507             PREFETCH" 32(%%"REG_c")             \n\t"
2508             PREFETCH" 64(%%"REG_c")             \n\t"
2509
2510 FUNNY_UV_CODE
2511 FUNNY_UV_CODE
2512 FUNNY_UV_CODE
2513 FUNNY_UV_CODE
2514
2515 #if defined(PIC)
2516             "mov %6, %%"REG_b"    \n\t"
2517 #endif
2518             :: "m" (src1), "m" (dst), "m" (mmx2Filter), "m" (mmx2FilterPos),
2519             "m" (funnyUVCode), "m" (src2)
2520 #if defined(PIC)
2521             ,"m" (ebxsave)
2522 #endif
2523             : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2524 #if !defined(PIC)
2525              ,"%"REG_b
2526 #endif
2527             );
2528             for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--)
2529             {
2530                 //printf("%d %d %d\n", dstWidth, i, srcW);
2531                 dst[i] = src1[srcW-1]*128;
2532                 dst[i+VOFW] = src2[srcW-1]*128;
2533             }
2534         }
2535         else
2536         {
2537 #endif /* HAVE_MMX2 */
2538             long xInc_shr16 = (long) (xInc >> 16);
2539             uint16_t xInc_mask = xInc & 0xffff;
2540             __asm__ volatile(
2541             "xor %%"REG_a", %%"REG_a"               \n\t" // i
2542             "xor %%"REG_d", %%"REG_d"               \n\t" // xx
2543             "xorl    %%ecx, %%ecx                   \n\t" // 2*xalpha
2544             ASMALIGN(4)
2545             "1:                                     \n\t"
2546             "mov        %0, %%"REG_S"               \n\t"
2547             "movzbl  (%%"REG_S", %%"REG_d"), %%edi  \n\t" //src[xx]
2548             "movzbl 1(%%"REG_S", %%"REG_d"), %%esi  \n\t" //src[xx+1]
2549             "subl    %%edi, %%esi                   \n\t" //src[xx+1] - src[xx]
2550             "imull   %%ecx, %%esi                   \n\t" //(src[xx+1] - src[xx])*2*xalpha
2551             "shll      $16, %%edi                   \n\t"
2552             "addl    %%edi, %%esi                   \n\t" //src[xx+1]*2*xalpha + src[xx]*(1-2*xalpha)
2553             "mov        %1, %%"REG_D"               \n\t"
2554             "shrl       $9, %%esi                   \n\t"
2555             "movw     %%si, (%%"REG_D", %%"REG_a", 2)   \n\t"
2556
2557             "movzbl    (%5, %%"REG_d"), %%edi       \n\t" //src[xx]
2558             "movzbl   1(%5, %%"REG_d"), %%esi       \n\t" //src[xx+1]
2559             "subl    %%edi, %%esi                   \n\t" //src[xx+1] - src[xx]
2560             "imull   %%ecx, %%esi                   \n\t" //(src[xx+1] - src[xx])*2*xalpha
2561             "shll      $16, %%edi                   \n\t"
2562             "addl    %%edi, %%esi                   \n\t" //src[xx+1]*2*xalpha + src[xx]*(1-2*xalpha)
2563             "mov        %1, %%"REG_D"               \n\t"
2564             "shrl       $9, %%esi                   \n\t"
2565             "movw     %%si, "AV_STRINGIFY(VOF)"(%%"REG_D", %%"REG_a", 2)   \n\t"
2566
2567             "addw       %4, %%cx                    \n\t" //2*xalpha += xInc&0xFF
2568             "adc        %3, %%"REG_d"               \n\t" //xx+= xInc>>8 + carry
2569             "add        $1, %%"REG_a"               \n\t"
2570             "cmp        %2, %%"REG_a"               \n\t"
2571             " jb        1b                          \n\t"
2572
2573 /* GCC 3.3 makes MPlayer crash on IA-32 machines when using "g" operand here,
2574    which is needed to support GCC 4.0. */
2575 #if defined(ARCH_X86_64) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4))
2576             :: "m" (src1), "m" (dst), "g" ((long)dstWidth), "m" (xInc_shr16), "m" (xInc_mask),
2577 #else
2578             :: "m" (src1), "m" (dst), "m" ((long)dstWidth), "m" (xInc_shr16), "m" (xInc_mask),
2579 #endif
2580             "r" (src2)
2581             : "%"REG_a, "%"REG_d, "%ecx", "%"REG_D, "%esi"
2582             );
2583 #ifdef HAVE_MMX2
2584         } //if MMX2 can't be used
2585 #endif
2586 #else
2587         int i;
2588         unsigned int xpos=0;
2589         for (i=0;i<dstWidth;i++)
2590         {
2591             register unsigned int xx=xpos>>16;
2592             register unsigned int xalpha=(xpos&0xFFFF)>>9;
2593             dst[i]=(src1[xx]*(xalpha^127)+src1[xx+1]*xalpha);
2594             dst[i+VOFW]=(src2[xx]*(xalpha^127)+src2[xx+1]*xalpha);
2595             /* slower
2596             dst[i]= (src1[xx]<<7) + (src1[xx+1] - src1[xx])*xalpha;
2597             dst[i+VOFW]=(src2[xx]<<7) + (src2[xx+1] - src2[xx])*xalpha;
2598             */
2599             xpos+=xInc;
2600         }
2601 #endif /* defined(ARCH_X86) */
2602     }
2603     if(c->srcRange != c->dstRange && !(isRGB(c->dstFormat) || isBGR(c->dstFormat))){
2604         int i;
2605         //FIXME all pal and rgb srcFormats could do this convertion as well
2606         //FIXME all scalers more complex than bilinear could do half of this transform
2607         if(c->srcRange){
2608             for (i=0; i<dstWidth; i++){
2609                 dst[i     ]= (dst[i     ]*1799 + 4081085)>>11; //1469
2610                 dst[i+VOFW]= (dst[i+VOFW]*1799 + 4081085)>>11; //1469
2611             }
2612         }else{
2613             for (i=0; i<dstWidth; i++){
2614                 dst[i     ]= (FFMIN(dst[i     ],30775)*4663 - 9289992)>>12; //-264
2615                 dst[i+VOFW]= (FFMIN(dst[i+VOFW],30775)*4663 - 9289992)>>12; //-264
2616             }
2617         }
2618     }
2619 }
2620
2621 static int RENAME(swScale)(SwsContext *c, uint8_t* src[], int srcStride[], int srcSliceY,
2622                            int srcSliceH, uint8_t* dst[], int dstStride[]){
2623
2624     /* load a few things into local vars to make the code more readable? and faster */
2625     const int srcW= c->srcW;
2626     const int dstW= c->dstW;
2627     const int dstH= c->dstH;
2628     const int chrDstW= c->chrDstW;
2629     const int chrSrcW= c->chrSrcW;
2630     const int lumXInc= c->lumXInc;
2631     const int chrXInc= c->chrXInc;
2632     const int dstFormat= c->dstFormat;
2633     const int srcFormat= c->srcFormat;
2634     const int flags= c->flags;
2635     const int canMMX2BeUsed= c->canMMX2BeUsed;
2636     int16_t *vLumFilterPos= c->vLumFilterPos;
2637     int16_t *vChrFilterPos= c->vChrFilterPos;
2638     int16_t *hLumFilterPos= c->hLumFilterPos;
2639     int16_t *hChrFilterPos= c->hChrFilterPos;
2640     int16_t *vLumFilter= c->vLumFilter;
2641     int16_t *vChrFilter= c->vChrFilter;
2642     int16_t *hLumFilter= c->hLumFilter;
2643     int16_t *hChrFilter= c->hChrFilter;
2644     int32_t *lumMmxFilter= c->lumMmxFilter;
2645     int32_t *chrMmxFilter= c->chrMmxFilter;
2646     const int vLumFilterSize= c->vLumFilterSize;
2647     const int vChrFilterSize= c->vChrFilterSize;
2648     const int hLumFilterSize= c->hLumFilterSize;
2649     const int hChrFilterSize= c->hChrFilterSize;
2650     int16_t **lumPixBuf= c->lumPixBuf;
2651     int16_t **chrPixBuf= c->chrPixBuf;
2652     const int vLumBufSize= c->vLumBufSize;
2653     const int vChrBufSize= c->vChrBufSize;
2654     uint8_t *funnyYCode= c->funnyYCode;
2655     uint8_t *funnyUVCode= c->funnyUVCode;
2656     uint8_t *formatConvBuffer= c->formatConvBuffer;
2657     const int chrSrcSliceY= srcSliceY >> c->chrSrcVSubSample;
2658     const int chrSrcSliceH= -((-srcSliceH) >> c->chrSrcVSubSample);
2659     int lastDstY;
2660     uint32_t *pal=c->pal_yuv;
2661
2662     /* vars which will change and which we need to store back in the context */
2663     int dstY= c->dstY;
2664     int lumBufIndex= c->lumBufIndex;
2665     int chrBufIndex= c->chrBufIndex;
2666     int lastInLumBuf= c->lastInLumBuf;
2667     int lastInChrBuf= c->lastInChrBuf;
2668
2669     if (isPacked(c->srcFormat)){
2670         src[0]=
2671         src[1]=
2672         src[2]= src[0];
2673         srcStride[0]=
2674         srcStride[1]=
2675         srcStride[2]= srcStride[0];
2676     }
2677     srcStride[1]<<= c->vChrDrop;
2678     srcStride[2]<<= c->vChrDrop;
2679
2680     //printf("swscale %X %X %X -> %X %X %X\n", (int)src[0], (int)src[1], (int)src[2],
2681     //       (int)dst[0], (int)dst[1], (int)dst[2]);
2682
2683 #if 0 //self test FIXME move to a vfilter or something
2684     {
2685     static volatile int i=0;
2686     i++;
2687     if (srcFormat==PIX_FMT_YUV420P && i==1 && srcSliceH>= c->srcH)
2688         selfTest(src, srcStride, c->srcW, c->srcH);
2689     i--;
2690     }
2691 #endif
2692
2693     //printf("sws Strides:%d %d %d -> %d %d %d\n", srcStride[0],srcStride[1],srcStride[2],
2694     //dstStride[0],dstStride[1],dstStride[2]);
2695
2696     if (dstStride[0]%8 !=0 || dstStride[1]%8 !=0 || dstStride[2]%8 !=0)
2697     {
2698         static int warnedAlready=0; //FIXME move this into the context perhaps
2699         if (flags & SWS_PRINT_INFO && !warnedAlready)
2700         {
2701             av_log(c, AV_LOG_WARNING, "Warning: dstStride is not aligned!\n"
2702                    "         ->cannot do aligned memory accesses anymore\n");
2703             warnedAlready=1;
2704         }
2705     }
2706
2707     /* Note the user might start scaling the picture in the middle so this
2708        will not get executed. This is not really intended but works
2709        currently, so people might do it. */
2710     if (srcSliceY ==0){
2711         lumBufIndex=0;
2712         chrBufIndex=0;
2713         dstY=0;
2714         lastInLumBuf= -1;
2715         lastInChrBuf= -1;
2716     }
2717
2718     lastDstY= dstY;
2719
2720     for (;dstY < dstH; dstY++){
2721         unsigned char *dest =dst[0]+dstStride[0]*dstY;
2722         const int chrDstY= dstY>>c->chrDstVSubSample;
2723         unsigned char *uDest=dst[1]+dstStride[1]*chrDstY;
2724         unsigned char *vDest=dst[2]+dstStride[2]*chrDstY;
2725
2726         const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input
2727         const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input
2728         const int lastLumSrcY= firstLumSrcY + vLumFilterSize -1; // Last line needed as input
2729         const int lastChrSrcY= firstChrSrcY + vChrFilterSize -1; // Last line needed as input
2730
2731         //printf("dstY:%d dstH:%d firstLumSrcY:%d lastInLumBuf:%d vLumBufSize: %d vChrBufSize: %d slice: %d %d vLumFilterSize: %d firstChrSrcY: %d vChrFilterSize: %d c->chrSrcVSubSample: %d\n",
2732         // dstY, dstH, firstLumSrcY, lastInLumBuf, vLumBufSize, vChrBufSize, srcSliceY, srcSliceH, vLumFilterSize, firstChrSrcY, vChrFilterSize,  c->chrSrcVSubSample);
2733         //handle holes (FAST_BILINEAR & weird filters)
2734         if (firstLumSrcY > lastInLumBuf) lastInLumBuf= firstLumSrcY-1;
2735         if (firstChrSrcY > lastInChrBuf) lastInChrBuf= firstChrSrcY-1;
2736         //printf("%d %d %d\n", firstChrSrcY, lastInChrBuf, vChrBufSize);
2737         assert(firstLumSrcY >= lastInLumBuf - vLumBufSize + 1);
2738         assert(firstChrSrcY >= lastInChrBuf - vChrBufSize + 1);
2739
2740         // Do we have enough lines in this slice to output the dstY line
2741         if (lastLumSrcY < srcSliceY + srcSliceH && lastChrSrcY < -((-srcSliceY - srcSliceH)>>c->chrSrcVSubSample))
2742         {
2743             //Do horizontal scaling
2744             while(lastInLumBuf < lastLumSrcY)
2745             {
2746                 uint8_t *s= src[0]+(lastInLumBuf + 1 - srcSliceY)*srcStride[0];
2747                 lumBufIndex++;
2748                 //printf("%d %d %d %d\n", lumBufIndex, vLumBufSize, lastInLumBuf,  lastLumSrcY);
2749                 assert(lumBufIndex < 2*vLumBufSize);
2750                 assert(lastInLumBuf + 1 - srcSliceY < srcSliceH);
2751                 assert(lastInLumBuf + 1 - srcSliceY >= 0);
2752                 //printf("%d %d\n", lumBufIndex, vLumBufSize);
2753                 RENAME(hyscale)(c, lumPixBuf[ lumBufIndex ], dstW, s, srcW, lumXInc,
2754                                 flags, canMMX2BeUsed, hLumFilter, hLumFilterPos, hLumFilterSize,
2755                                 funnyYCode, c->srcFormat, formatConvBuffer,
2756                                 c->lumMmx2Filter, c->lumMmx2FilterPos, pal);
2757                 lastInLumBuf++;
2758             }
2759             while(lastInChrBuf < lastChrSrcY)
2760             {
2761                 uint8_t *src1= src[1]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[1];
2762                 uint8_t *src2= src[2]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[2];
2763                 chrBufIndex++;
2764                 assert(chrBufIndex < 2*vChrBufSize);
2765                 assert(lastInChrBuf + 1 - chrSrcSliceY < (chrSrcSliceH));
2766                 assert(lastInChrBuf + 1 - chrSrcSliceY >= 0);
2767                 //FIXME replace parameters through context struct (some at least)
2768
2769                 if (!(isGray(srcFormat) || isGray(dstFormat)))
2770                     RENAME(hcscale)(c, chrPixBuf[ chrBufIndex ], chrDstW, src1, src2, chrSrcW, chrXInc,
2771                                     flags, canMMX2BeUsed, hChrFilter, hChrFilterPos, hChrFilterSize,
2772                                     funnyUVCode, c->srcFormat, formatConvBuffer,
2773                                     c->chrMmx2Filter, c->chrMmx2FilterPos, pal);
2774                 lastInChrBuf++;
2775             }
2776             //wrap buf index around to stay inside the ring buffer
2777             if (lumBufIndex >= vLumBufSize) lumBufIndex-= vLumBufSize;
2778             if (chrBufIndex >= vChrBufSize) chrBufIndex-= vChrBufSize;
2779         }
2780         else // not enough lines left in this slice -> load the rest in the buffer
2781         {
2782             /* printf("%d %d Last:%d %d LastInBuf:%d %d Index:%d %d Y:%d FSize: %d %d BSize: %d %d\n",
2783             firstChrSrcY,firstLumSrcY,lastChrSrcY,lastLumSrcY,
2784             lastInChrBuf,lastInLumBuf,chrBufIndex,lumBufIndex,dstY,vChrFilterSize,vLumFilterSize,
2785             vChrBufSize, vLumBufSize);*/
2786
2787             //Do horizontal scaling
2788             while(lastInLumBuf+1 < srcSliceY + srcSliceH)
2789             {
2790                 uint8_t *s= src[0]+(lastInLumBuf + 1 - srcSliceY)*srcStride[0];
2791                 lumBufIndex++;
2792                 assert(lumBufIndex < 2*vLumBufSize);
2793                 assert(lastInLumBuf + 1 - srcSliceY < srcSliceH);
2794                 assert(lastInLumBuf + 1 - srcSliceY >= 0);
2795                 RENAME(hyscale)(c, lumPixBuf[ lumBufIndex ], dstW, s, srcW, lumXInc,
2796                                 flags, canMMX2BeUsed, hLumFilter, hLumFilterPos, hLumFilterSize,
2797                                 funnyYCode, c->srcFormat, formatConvBuffer,
2798                                 c->lumMmx2Filter, c->lumMmx2FilterPos, pal);
2799                 lastInLumBuf++;
2800             }
2801             while(lastInChrBuf+1 < (chrSrcSliceY + chrSrcSliceH))
2802             {
2803                 uint8_t *src1= src[1]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[1];
2804                 uint8_t *src2= src[2]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[2];
2805                 chrBufIndex++;
2806                 assert(chrBufIndex < 2*vChrBufSize);
2807                 assert(lastInChrBuf + 1 - chrSrcSliceY < chrSrcSliceH);
2808                 assert(lastInChrBuf + 1 - chrSrcSliceY >= 0);
2809
2810                 if (!(isGray(srcFormat) || isGray(dstFormat)))
2811                     RENAME(hcscale)(c, chrPixBuf[ chrBufIndex ], chrDstW, src1, src2, chrSrcW, chrXInc,
2812                             flags, canMMX2BeUsed, hChrFilter, hChrFilterPos, hChrFilterSize,
2813                             funnyUVCode, c->srcFormat, formatConvBuffer,
2814                             c->chrMmx2Filter, c->chrMmx2FilterPos, pal);
2815                 lastInChrBuf++;
2816             }
2817             //wrap buf index around to stay inside the ring buffer
2818             if (lumBufIndex >= vLumBufSize) lumBufIndex-= vLumBufSize;
2819             if (chrBufIndex >= vChrBufSize) chrBufIndex-= vChrBufSize;
2820             break; //we can't output a dstY line so let's try with the next slice
2821         }
2822
2823 #ifdef HAVE_MMX
2824         c->blueDither= ff_dither8[dstY&1];
2825         if (c->dstFormat == PIX_FMT_RGB555 || c->dstFormat == PIX_FMT_BGR555)
2826             c->greenDither= ff_dither8[dstY&1];
2827         else
2828             c->greenDither= ff_dither4[dstY&1];
2829         c->redDither= ff_dither8[(dstY+1)&1];
2830 #endif
2831         if (dstY < dstH-2)
2832         {
2833             int16_t **lumSrcPtr= lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2834             int16_t **chrSrcPtr= chrPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2835 #ifdef HAVE_MMX
2836             int i;
2837         if (flags & SWS_ACCURATE_RND){
2838             int s= APCK_SIZE / 8;
2839             for (i=0; i<vLumFilterSize; i+=2){
2840                 *(void**)&lumMmxFilter[s*i              ]= lumSrcPtr[i  ];
2841                 *(void**)&lumMmxFilter[s*i+APCK_PTR2/4  ]= lumSrcPtr[i+(vLumFilterSize>1)];
2842                           lumMmxFilter[s*i+APCK_COEF/4  ]=
2843                           lumMmxFilter[s*i+APCK_COEF/4+1]= vLumFilter[dstY*vLumFilterSize + i    ]
2844                     + (vLumFilterSize>1 ? vLumFilter[dstY*vLumFilterSize + i + 1]<<16 : 0);
2845             }
2846             for (i=0; i<vChrFilterSize; i+=2){
2847                 *(void**)&chrMmxFilter[s*i              ]= chrSrcPtr[i  ];
2848                 *(void**)&chrMmxFilter[s*i+APCK_PTR2/4  ]= chrSrcPtr[i+(vChrFilterSize>1)];
2849                           chrMmxFilter[s*i+APCK_COEF/4  ]=
2850                           chrMmxFilter[s*i+APCK_COEF/4+1]= vChrFilter[chrDstY*vChrFilterSize + i    ]
2851                     + (vChrFilterSize>1 ? vChrFilter[chrDstY*vChrFilterSize + i + 1]<<16 : 0);
2852             }
2853         }else{
2854             for (i=0; i<vLumFilterSize; i++)
2855             {
2856                 lumMmxFilter[4*i+0]= (int32_t)lumSrcPtr[i];
2857                 lumMmxFilter[4*i+1]= (uint64_t)lumSrcPtr[i] >> 32;
2858                 lumMmxFilter[4*i+2]=
2859                 lumMmxFilter[4*i+3]=
2860                     ((uint16_t)vLumFilter[dstY*vLumFilterSize + i])*0x10001;
2861             }
2862             for (i=0; i<vChrFilterSize; i++)
2863             {
2864                 chrMmxFilter[4*i+0]= (int32_t)chrSrcPtr[i];
2865                 chrMmxFilter[4*i+1]= (uint64_t)chrSrcPtr[i] >> 32;
2866                 chrMmxFilter[4*i+2]=
2867                 chrMmxFilter[4*i+3]=
2868                     ((uint16_t)vChrFilter[chrDstY*vChrFilterSize + i])*0x10001;
2869             }
2870         }
2871 #endif
2872             if (dstFormat == PIX_FMT_NV12 || dstFormat == PIX_FMT_NV21){
2873                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2874                 if (dstY&chrSkipMask) uDest= NULL; //FIXME split functions in lumi / chromi
2875                 RENAME(yuv2nv12X)(c,
2876                     vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2877                     vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2878                     dest, uDest, dstW, chrDstW, dstFormat);
2879             }
2880             else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) //YV12 like
2881             {
2882                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2883                 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
2884                 if (vLumFilterSize == 1 && vChrFilterSize == 1) // unscaled YV12
2885                 {
2886                     int16_t *lumBuf = lumPixBuf[0];
2887                     int16_t *chrBuf= chrPixBuf[0];
2888                     RENAME(yuv2yuv1)(c, lumBuf, chrBuf, dest, uDest, vDest, dstW, chrDstW);
2889                 }
2890                 else //General YV12
2891                 {
2892                     RENAME(yuv2yuvX)(c,
2893                         vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2894                         vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2895                         dest, uDest, vDest, dstW, chrDstW);
2896                 }
2897             }
2898             else
2899             {
2900                 assert(lumSrcPtr + vLumFilterSize - 1 < lumPixBuf + vLumBufSize*2);
2901                 assert(chrSrcPtr + vChrFilterSize - 1 < chrPixBuf + vChrBufSize*2);
2902                 if (vLumFilterSize == 1 && vChrFilterSize == 2) //unscaled RGB
2903                 {
2904                     int chrAlpha= vChrFilter[2*dstY+1];
2905                     if(flags & SWS_FULL_CHR_H_INT){
2906                         yuv2rgbXinC_full(c, //FIXME write a packed1_full function
2907                             vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2908                             vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2909                             dest, dstW, dstY);
2910                     }else{
2911                         RENAME(yuv2packed1)(c, *lumSrcPtr, *chrSrcPtr, *(chrSrcPtr+1),
2912                             dest, dstW, chrAlpha, dstFormat, flags, dstY);
2913                     }
2914                 }
2915                 else if (vLumFilterSize == 2 && vChrFilterSize == 2) //bilinear upscale RGB
2916                 {
2917                     int lumAlpha= vLumFilter[2*dstY+1];
2918                     int chrAlpha= vChrFilter[2*dstY+1];
2919                     lumMmxFilter[2]=
2920                     lumMmxFilter[3]= vLumFilter[2*dstY   ]*0x10001;
2921                     chrMmxFilter[2]=
2922                     chrMmxFilter[3]= vChrFilter[2*chrDstY]*0x10001;
2923                     if(flags & SWS_FULL_CHR_H_INT){
2924                         yuv2rgbXinC_full(c, //FIXME write a packed2_full function
2925                             vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2926                             vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2927                             dest, dstW, dstY);
2928                     }else{
2929                         RENAME(yuv2packed2)(c, *lumSrcPtr, *(lumSrcPtr+1), *chrSrcPtr, *(chrSrcPtr+1),
2930                             dest, dstW, lumAlpha, chrAlpha, dstY);
2931                     }
2932                 }
2933                 else //general RGB
2934                 {
2935                     if(flags & SWS_FULL_CHR_H_INT){
2936                         yuv2rgbXinC_full(c,
2937                             vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2938                             vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2939                             dest, dstW, dstY);
2940                     }else{
2941                         RENAME(yuv2packedX)(c,
2942                             vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2943                             vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2944                             dest, dstW, dstY);
2945                     }
2946                 }
2947             }
2948         }
2949         else // hmm looks like we can't use MMX here without overwriting this array's tail
2950         {
2951             int16_t **lumSrcPtr= lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2952             int16_t **chrSrcPtr= chrPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2953             if (dstFormat == PIX_FMT_NV12 || dstFormat == PIX_FMT_NV21){
2954                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2955                 if (dstY&chrSkipMask) uDest= NULL; //FIXME split functions in lumi / chromi
2956                 yuv2nv12XinC(
2957                     vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2958                     vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2959                     dest, uDest, dstW, chrDstW, dstFormat);
2960             }
2961             else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) //YV12
2962             {
2963                 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2964                 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
2965                 yuv2yuvXinC(
2966                     vLumFilter+dstY*vLumFilterSize   , lumSrcPtr, vLumFilterSize,
2967                     vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2968                     dest, uDest, vDest, dstW, chrDstW);
2969             }
2970             else
2971             {
2972                 assert(lumSrcPtr + vLumFilterSize - 1 < lumPixBuf + vLumBufSize*2);
2973                 assert(chrSrcPtr + vChrFilterSize - 1 < chrPixBuf + vChrBufSize*2);
2974                 if(flags & SWS_FULL_CHR_H_INT){
2975                     yuv2rgbXinC_full(c,
2976                         vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2977                         vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2978                         dest, dstW, dstY);
2979                 }else{
2980                     yuv2packedXinC(c,
2981                         vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2982                         vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2983                         dest, dstW, dstY);
2984                 }
2985             }
2986         }
2987     }
2988
2989 #ifdef HAVE_MMX
2990     __asm__ volatile(SFENCE:::"memory");
2991     __asm__ volatile(EMMS:::"memory");
2992 #endif
2993     /* store changed local vars back in the context */
2994     c->dstY= dstY;
2995     c->lumBufIndex= lumBufIndex;
2996     c->chrBufIndex= chrBufIndex;
2997     c->lastInLumBuf= lastInLumBuf;
2998     c->lastInChrBuf= lastInChrBuf;
2999
3000     return dstY - lastDstY;
3001 }