New implementation of rgb32tobgr32
[ffmpeg.git] / libswscale / rgb2rgb_template.c
1 /*
2  *
3  *  rgb2rgb.c, Software RGB to RGB convertor
4  *  pluralize by Software PAL8 to RGB convertor
5  *               Software YUV to YUV convertor
6  *               Software YUV to RGB convertor
7  *  Written by Nick Kurshev.
8  *  palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
9  *  lot of big-endian byteorder fixes by Alex Beregszaszi
10  *
11  * This file is part of FFmpeg.
12  *
13  * FFmpeg is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  * FFmpeg is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21  * GNU General Public License for more details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with FFmpeg; if not, write to the Free Software
25  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26  * 
27  * the C code (not assembly, mmx, ...) of this file can be used
28  * under the LGPL license too
29  */
30
31 #include <stddef.h>
32 #include <inttypes.h> /* for __WORDSIZE */
33
34 #ifndef __WORDSIZE
35 // #warning You have misconfigured system and probably will lose performance!
36 #define __WORDSIZE MP_WORDSIZE
37 #endif
38
39 #undef PREFETCH
40 #undef MOVNTQ
41 #undef EMMS
42 #undef SFENCE
43 #undef MMREG_SIZE
44 #undef PREFETCHW
45 #undef PAVGB
46
47 #ifdef HAVE_SSE2
48 #define MMREG_SIZE 16
49 #else
50 #define MMREG_SIZE 8
51 #endif
52
53 #ifdef HAVE_3DNOW
54 #define PREFETCH  "prefetch"
55 #define PREFETCHW "prefetchw"
56 #define PAVGB     "pavgusb"
57 #elif defined ( HAVE_MMX2 )
58 #define PREFETCH "prefetchnta"
59 #define PREFETCHW "prefetcht0"
60 #define PAVGB     "pavgb"
61 #else
62 #ifdef __APPLE__
63 #define PREFETCH "#"
64 #define PREFETCHW "#"
65 #else
66 #define PREFETCH  " # nop"
67 #define PREFETCHW " # nop"
68 #endif
69 #endif
70
71 #ifdef HAVE_3DNOW
72 /* On K6 femms is faster of emms. On K7 femms is directly mapped on emms. */
73 #define EMMS     "femms"
74 #else
75 #define EMMS     "emms"
76 #endif
77
78 #ifdef HAVE_MMX2
79 #define MOVNTQ "movntq"
80 #define SFENCE "sfence"
81 #else
82 #define MOVNTQ "movq"
83 #define SFENCE " # nop"
84 #endif
85
86 static inline void RENAME(rgb24to32)(const uint8_t *src,uint8_t *dst,long src_size)
87 {
88   uint8_t *dest = dst;
89   const uint8_t *s = src;
90   const uint8_t *end;
91 #ifdef HAVE_MMX
92   const uint8_t *mm_end;
93 #endif
94   end = s + src_size;
95 #ifdef HAVE_MMX
96   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
97   mm_end = end - 23;
98   __asm __volatile("movq        %0, %%mm7"::"m"(mask32):"memory");
99   while(s < mm_end)
100   {
101     __asm __volatile(
102         PREFETCH"       32%1\n\t"
103         "movd   %1, %%mm0\n\t"
104         "punpckldq 3%1, %%mm0\n\t"
105         "movd   6%1, %%mm1\n\t"
106         "punpckldq 9%1, %%mm1\n\t"
107         "movd   12%1, %%mm2\n\t"
108         "punpckldq 15%1, %%mm2\n\t"
109         "movd   18%1, %%mm3\n\t"
110         "punpckldq 21%1, %%mm3\n\t"
111         "pand   %%mm7, %%mm0\n\t"
112         "pand   %%mm7, %%mm1\n\t"
113         "pand   %%mm7, %%mm2\n\t"
114         "pand   %%mm7, %%mm3\n\t"
115         MOVNTQ" %%mm0, %0\n\t"
116         MOVNTQ" %%mm1, 8%0\n\t"
117         MOVNTQ" %%mm2, 16%0\n\t"
118         MOVNTQ" %%mm3, 24%0"
119         :"=m"(*dest)
120         :"m"(*s)
121         :"memory");
122     dest += 32;
123     s += 24;
124   }
125   __asm __volatile(SFENCE:::"memory");
126   __asm __volatile(EMMS:::"memory");
127 #endif
128   while(s < end)
129   {
130 #ifdef WORDS_BIGENDIAN
131     /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */
132     *dest++ = 0;
133     *dest++ = s[2];
134     *dest++ = s[1];
135     *dest++ = s[0];
136     s+=3;
137 #else
138     *dest++ = *s++;
139     *dest++ = *s++;
140     *dest++ = *s++;
141     *dest++ = 0;
142 #endif
143   }
144 }
145
146 static inline void RENAME(rgb32to24)(const uint8_t *src,uint8_t *dst,long src_size)
147 {
148   uint8_t *dest = dst;
149   const uint8_t *s = src;
150   const uint8_t *end;
151 #ifdef HAVE_MMX
152   const uint8_t *mm_end;
153 #endif
154   end = s + src_size;
155 #ifdef HAVE_MMX
156   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
157   mm_end = end - 31;
158   while(s < mm_end)
159   {
160     __asm __volatile(
161         PREFETCH"       32%1\n\t"
162         "movq   %1, %%mm0\n\t"
163         "movq   8%1, %%mm1\n\t"
164         "movq   16%1, %%mm4\n\t"
165         "movq   24%1, %%mm5\n\t"
166         "movq   %%mm0, %%mm2\n\t"
167         "movq   %%mm1, %%mm3\n\t"
168         "movq   %%mm4, %%mm6\n\t"
169         "movq   %%mm5, %%mm7\n\t"
170         "psrlq  $8, %%mm2\n\t"
171         "psrlq  $8, %%mm3\n\t"
172         "psrlq  $8, %%mm6\n\t"
173         "psrlq  $8, %%mm7\n\t"
174         "pand   %2, %%mm0\n\t"
175         "pand   %2, %%mm1\n\t"
176         "pand   %2, %%mm4\n\t"
177         "pand   %2, %%mm5\n\t"
178         "pand   %3, %%mm2\n\t"
179         "pand   %3, %%mm3\n\t"
180         "pand   %3, %%mm6\n\t"
181         "pand   %3, %%mm7\n\t"
182         "por    %%mm2, %%mm0\n\t"
183         "por    %%mm3, %%mm1\n\t"
184         "por    %%mm6, %%mm4\n\t"
185         "por    %%mm7, %%mm5\n\t"
186
187         "movq   %%mm1, %%mm2\n\t"
188         "movq   %%mm4, %%mm3\n\t"
189         "psllq  $48, %%mm2\n\t"
190         "psllq  $32, %%mm3\n\t"
191         "pand   %4, %%mm2\n\t"
192         "pand   %5, %%mm3\n\t"
193         "por    %%mm2, %%mm0\n\t"
194         "psrlq  $16, %%mm1\n\t"
195         "psrlq  $32, %%mm4\n\t"
196         "psllq  $16, %%mm5\n\t"
197         "por    %%mm3, %%mm1\n\t"
198         "pand   %6, %%mm5\n\t"
199         "por    %%mm5, %%mm4\n\t"
200
201         MOVNTQ" %%mm0, %0\n\t"
202         MOVNTQ" %%mm1, 8%0\n\t"
203         MOVNTQ" %%mm4, 16%0"
204         :"=m"(*dest)
205         :"m"(*s),"m"(mask24l),
206          "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
207         :"memory");
208     dest += 24;
209     s += 32;
210   }
211   __asm __volatile(SFENCE:::"memory");
212   __asm __volatile(EMMS:::"memory");
213 #endif
214   while(s < end)
215   {
216 #ifdef WORDS_BIGENDIAN
217     /* RGB32 (= A,B,G,R) -> RGB24 (= R,G,B) */
218     s++;
219     dest[2] = *s++;
220     dest[1] = *s++;
221     dest[0] = *s++;
222     dest += 3;
223 #else
224     *dest++ = *s++;
225     *dest++ = *s++;
226     *dest++ = *s++;
227     s++;
228 #endif
229   }
230 }
231
232 /*
233  Original by Strepto/Astral
234  ported to gcc & bugfixed : A'rpi
235  MMX2, 3DNOW optimization by Nick Kurshev
236  32bit c version, and and&add trick by Michael Niedermayer
237 */
238 static inline void RENAME(rgb15to16)(const uint8_t *src,uint8_t *dst,long src_size)
239 {
240   register const uint8_t* s=src;
241   register uint8_t* d=dst;
242   register const uint8_t *end;
243   const uint8_t *mm_end;
244   end = s + src_size;
245 #ifdef HAVE_MMX
246   __asm __volatile(PREFETCH"    %0"::"m"(*s));
247   __asm __volatile("movq        %0, %%mm4"::"m"(mask15s));
248   mm_end = end - 15;
249   while(s<mm_end)
250   {
251         __asm __volatile(
252                 PREFETCH"       32%1\n\t"
253                 "movq   %1, %%mm0\n\t"
254                 "movq   8%1, %%mm2\n\t"
255                 "movq   %%mm0, %%mm1\n\t"
256                 "movq   %%mm2, %%mm3\n\t"
257                 "pand   %%mm4, %%mm0\n\t"
258                 "pand   %%mm4, %%mm2\n\t"
259                 "paddw  %%mm1, %%mm0\n\t"
260                 "paddw  %%mm3, %%mm2\n\t"
261                 MOVNTQ" %%mm0, %0\n\t"
262                 MOVNTQ" %%mm2, 8%0"
263                 :"=m"(*d)
264                 :"m"(*s)
265                 );
266         d+=16;
267         s+=16;
268   }
269   __asm __volatile(SFENCE:::"memory");
270   __asm __volatile(EMMS:::"memory");
271 #endif
272     mm_end = end - 3;
273     while(s < mm_end)
274     {
275         register unsigned x= *((uint32_t *)s);
276         *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
277         d+=4;
278         s+=4;
279     }
280     if(s < end)
281     {
282         register unsigned short x= *((uint16_t *)s);
283         *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
284     }
285 }
286
287 static inline void RENAME(rgb16to15)(const uint8_t *src,uint8_t *dst,long src_size)
288 {
289   register const uint8_t* s=src;
290   register uint8_t* d=dst;
291   register const uint8_t *end;
292   const uint8_t *mm_end;
293   end = s + src_size;
294 #ifdef HAVE_MMX
295   __asm __volatile(PREFETCH"    %0"::"m"(*s));
296   __asm __volatile("movq        %0, %%mm7"::"m"(mask15rg));
297   __asm __volatile("movq        %0, %%mm6"::"m"(mask15b));
298   mm_end = end - 15;
299   while(s<mm_end)
300   {
301         __asm __volatile(
302                 PREFETCH"       32%1\n\t"
303                 "movq   %1, %%mm0\n\t"
304                 "movq   8%1, %%mm2\n\t"
305                 "movq   %%mm0, %%mm1\n\t"
306                 "movq   %%mm2, %%mm3\n\t"
307                 "psrlq  $1, %%mm0\n\t"
308                 "psrlq  $1, %%mm2\n\t"
309                 "pand   %%mm7, %%mm0\n\t"
310                 "pand   %%mm7, %%mm2\n\t"
311                 "pand   %%mm6, %%mm1\n\t"
312                 "pand   %%mm6, %%mm3\n\t"
313                 "por    %%mm1, %%mm0\n\t"
314                 "por    %%mm3, %%mm2\n\t"
315                 MOVNTQ" %%mm0, %0\n\t"
316                 MOVNTQ" %%mm2, 8%0"
317                 :"=m"(*d)
318                 :"m"(*s)
319                 );
320         d+=16;
321         s+=16;
322   }
323   __asm __volatile(SFENCE:::"memory");
324   __asm __volatile(EMMS:::"memory");
325 #endif
326     mm_end = end - 3;
327     while(s < mm_end)
328     {
329         register uint32_t x= *((uint32_t *)s);
330         *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
331         s+=4;
332         d+=4;
333     }
334     if(s < end)
335     {
336         register uint16_t x= *((uint16_t *)s);
337         *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
338         s+=2;
339         d+=2;
340     }
341 }
342
343 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, long src_size)
344 {
345         const uint8_t *s = src;
346         const uint8_t *end;
347 #ifdef HAVE_MMX
348         const uint8_t *mm_end;
349 #endif
350         uint16_t *d = (uint16_t *)dst;
351         end = s + src_size;
352 #ifdef HAVE_MMX
353         mm_end = end - 15;
354 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
355         asm volatile(
356                 "movq %3, %%mm5                 \n\t"
357                 "movq %4, %%mm6                 \n\t"
358                 "movq %5, %%mm7                 \n\t"
359                 ASMALIGN(4)
360                 "1:                             \n\t"
361                 PREFETCH" 32(%1)                \n\t"
362                 "movd   (%1), %%mm0             \n\t"
363                 "movd   4(%1), %%mm3            \n\t"
364                 "punpckldq 8(%1), %%mm0         \n\t"
365                 "punpckldq 12(%1), %%mm3        \n\t"
366                 "movq %%mm0, %%mm1              \n\t"
367                 "movq %%mm3, %%mm4              \n\t"
368                 "pand %%mm6, %%mm0              \n\t"
369                 "pand %%mm6, %%mm3              \n\t"
370                 "pmaddwd %%mm7, %%mm0           \n\t"
371                 "pmaddwd %%mm7, %%mm3           \n\t"
372                 "pand %%mm5, %%mm1              \n\t"
373                 "pand %%mm5, %%mm4              \n\t"
374                 "por %%mm1, %%mm0               \n\t"   
375                 "por %%mm4, %%mm3               \n\t"
376                 "psrld $5, %%mm0                \n\t"
377                 "pslld $11, %%mm3               \n\t"
378                 "por %%mm3, %%mm0               \n\t"
379                 MOVNTQ" %%mm0, (%0)             \n\t"
380                 "add $16, %1                    \n\t"
381                 "add $8, %0                     \n\t"
382                 "cmp %2, %1                     \n\t"
383                 " jb 1b                         \n\t"
384                 : "+r" (d), "+r"(s)
385                 : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
386         );
387 #else
388         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
389         __asm __volatile(
390             "movq       %0, %%mm7\n\t"
391             "movq       %1, %%mm6\n\t"
392             ::"m"(red_16mask),"m"(green_16mask));
393         while(s < mm_end)
394         {
395             __asm __volatile(
396                 PREFETCH" 32%1\n\t"
397                 "movd   %1, %%mm0\n\t"
398                 "movd   4%1, %%mm3\n\t"
399                 "punpckldq 8%1, %%mm0\n\t"
400                 "punpckldq 12%1, %%mm3\n\t"
401                 "movq   %%mm0, %%mm1\n\t"
402                 "movq   %%mm0, %%mm2\n\t"
403                 "movq   %%mm3, %%mm4\n\t"
404                 "movq   %%mm3, %%mm5\n\t"
405                 "psrlq  $3, %%mm0\n\t"
406                 "psrlq  $3, %%mm3\n\t"
407                 "pand   %2, %%mm0\n\t"
408                 "pand   %2, %%mm3\n\t"
409                 "psrlq  $5, %%mm1\n\t"
410                 "psrlq  $5, %%mm4\n\t"
411                 "pand   %%mm6, %%mm1\n\t"
412                 "pand   %%mm6, %%mm4\n\t"
413                 "psrlq  $8, %%mm2\n\t"
414                 "psrlq  $8, %%mm5\n\t"
415                 "pand   %%mm7, %%mm2\n\t"
416                 "pand   %%mm7, %%mm5\n\t"
417                 "por    %%mm1, %%mm0\n\t"
418                 "por    %%mm4, %%mm3\n\t"
419                 "por    %%mm2, %%mm0\n\t"
420                 "por    %%mm5, %%mm3\n\t"
421                 "psllq  $16, %%mm3\n\t"
422                 "por    %%mm3, %%mm0\n\t"
423                 MOVNTQ" %%mm0, %0\n\t"
424                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
425                 d += 4;
426                 s += 16;
427         }
428 #endif
429         __asm __volatile(SFENCE:::"memory");
430         __asm __volatile(EMMS:::"memory");
431 #endif
432         while(s < end)
433         {
434                 register int rgb = *(uint32_t*)s; s += 4;
435                 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
436         }
437 }
438
439 static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
440 {
441         const uint8_t *s = src;
442         const uint8_t *end;
443 #ifdef HAVE_MMX
444         const uint8_t *mm_end;
445 #endif
446         uint16_t *d = (uint16_t *)dst;
447         end = s + src_size;
448 #ifdef HAVE_MMX
449         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
450         __asm __volatile(
451             "movq       %0, %%mm7\n\t"
452             "movq       %1, %%mm6\n\t"
453             ::"m"(red_16mask),"m"(green_16mask));
454         mm_end = end - 15;
455         while(s < mm_end)
456         {
457             __asm __volatile(
458                 PREFETCH" 32%1\n\t"
459                 "movd   %1, %%mm0\n\t"
460                 "movd   4%1, %%mm3\n\t"
461                 "punpckldq 8%1, %%mm0\n\t"
462                 "punpckldq 12%1, %%mm3\n\t"
463                 "movq   %%mm0, %%mm1\n\t"
464                 "movq   %%mm0, %%mm2\n\t"
465                 "movq   %%mm3, %%mm4\n\t"
466                 "movq   %%mm3, %%mm5\n\t"
467                 "psllq  $8, %%mm0\n\t"
468                 "psllq  $8, %%mm3\n\t"
469                 "pand   %%mm7, %%mm0\n\t"
470                 "pand   %%mm7, %%mm3\n\t"
471                 "psrlq  $5, %%mm1\n\t"
472                 "psrlq  $5, %%mm4\n\t"
473                 "pand   %%mm6, %%mm1\n\t"
474                 "pand   %%mm6, %%mm4\n\t"
475                 "psrlq  $19, %%mm2\n\t"
476                 "psrlq  $19, %%mm5\n\t"
477                 "pand   %2, %%mm2\n\t"
478                 "pand   %2, %%mm5\n\t"
479                 "por    %%mm1, %%mm0\n\t"
480                 "por    %%mm4, %%mm3\n\t"
481                 "por    %%mm2, %%mm0\n\t"
482                 "por    %%mm5, %%mm3\n\t"
483                 "psllq  $16, %%mm3\n\t"
484                 "por    %%mm3, %%mm0\n\t"
485                 MOVNTQ" %%mm0, %0\n\t"
486                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
487                 d += 4;
488                 s += 16;
489         }
490         __asm __volatile(SFENCE:::"memory");
491         __asm __volatile(EMMS:::"memory");
492 #endif
493         while(s < end)
494         {
495                 register int rgb = *(uint32_t*)s; s += 4;
496                 *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
497         }
498 }
499
500 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, long src_size)
501 {
502         const uint8_t *s = src;
503         const uint8_t *end;
504 #ifdef HAVE_MMX
505         const uint8_t *mm_end;
506 #endif
507         uint16_t *d = (uint16_t *)dst;
508         end = s + src_size;
509 #ifdef HAVE_MMX
510         mm_end = end - 15;
511 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
512         asm volatile(
513                 "movq %3, %%mm5                 \n\t"
514                 "movq %4, %%mm6                 \n\t"
515                 "movq %5, %%mm7                 \n\t"
516                 ASMALIGN(4)
517                 "1:                             \n\t"
518                 PREFETCH" 32(%1)                \n\t"
519                 "movd   (%1), %%mm0             \n\t"
520                 "movd   4(%1), %%mm3            \n\t"
521                 "punpckldq 8(%1), %%mm0         \n\t"
522                 "punpckldq 12(%1), %%mm3        \n\t"
523                 "movq %%mm0, %%mm1              \n\t"
524                 "movq %%mm3, %%mm4              \n\t"
525                 "pand %%mm6, %%mm0              \n\t"
526                 "pand %%mm6, %%mm3              \n\t"
527                 "pmaddwd %%mm7, %%mm0           \n\t"
528                 "pmaddwd %%mm7, %%mm3           \n\t"
529                 "pand %%mm5, %%mm1              \n\t"
530                 "pand %%mm5, %%mm4              \n\t"
531                 "por %%mm1, %%mm0               \n\t"   
532                 "por %%mm4, %%mm3               \n\t"
533                 "psrld $6, %%mm0                \n\t"
534                 "pslld $10, %%mm3               \n\t"
535                 "por %%mm3, %%mm0               \n\t"
536                 MOVNTQ" %%mm0, (%0)             \n\t"
537                 "add $16, %1                    \n\t"
538                 "add $8, %0                     \n\t"
539                 "cmp %2, %1                     \n\t"
540                 " jb 1b                         \n\t"
541                 : "+r" (d), "+r"(s)
542                 : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
543         );
544 #else
545         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
546         __asm __volatile(
547             "movq       %0, %%mm7\n\t"
548             "movq       %1, %%mm6\n\t"
549             ::"m"(red_15mask),"m"(green_15mask));
550         while(s < mm_end)
551         {
552             __asm __volatile(
553                 PREFETCH" 32%1\n\t"
554                 "movd   %1, %%mm0\n\t"
555                 "movd   4%1, %%mm3\n\t"
556                 "punpckldq 8%1, %%mm0\n\t"
557                 "punpckldq 12%1, %%mm3\n\t"
558                 "movq   %%mm0, %%mm1\n\t"
559                 "movq   %%mm0, %%mm2\n\t"
560                 "movq   %%mm3, %%mm4\n\t"
561                 "movq   %%mm3, %%mm5\n\t"
562                 "psrlq  $3, %%mm0\n\t"
563                 "psrlq  $3, %%mm3\n\t"
564                 "pand   %2, %%mm0\n\t"
565                 "pand   %2, %%mm3\n\t"
566                 "psrlq  $6, %%mm1\n\t"
567                 "psrlq  $6, %%mm4\n\t"
568                 "pand   %%mm6, %%mm1\n\t"
569                 "pand   %%mm6, %%mm4\n\t"
570                 "psrlq  $9, %%mm2\n\t"
571                 "psrlq  $9, %%mm5\n\t"
572                 "pand   %%mm7, %%mm2\n\t"
573                 "pand   %%mm7, %%mm5\n\t"
574                 "por    %%mm1, %%mm0\n\t"
575                 "por    %%mm4, %%mm3\n\t"
576                 "por    %%mm2, %%mm0\n\t"
577                 "por    %%mm5, %%mm3\n\t"
578                 "psllq  $16, %%mm3\n\t"
579                 "por    %%mm3, %%mm0\n\t"
580                 MOVNTQ" %%mm0, %0\n\t"
581                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
582                 d += 4;
583                 s += 16;
584         }
585 #endif
586         __asm __volatile(SFENCE:::"memory");
587         __asm __volatile(EMMS:::"memory");
588 #endif
589         while(s < end)
590         {
591                 register int rgb = *(uint32_t*)s; s += 4;
592                 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
593         }
594 }
595
596 static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
597 {
598         const uint8_t *s = src;
599         const uint8_t *end;
600 #ifdef HAVE_MMX
601         const uint8_t *mm_end;
602 #endif
603         uint16_t *d = (uint16_t *)dst;
604         end = s + src_size;
605 #ifdef HAVE_MMX
606         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
607         __asm __volatile(
608             "movq       %0, %%mm7\n\t"
609             "movq       %1, %%mm6\n\t"
610             ::"m"(red_15mask),"m"(green_15mask));
611         mm_end = end - 15;
612         while(s < mm_end)
613         {
614             __asm __volatile(
615                 PREFETCH" 32%1\n\t"
616                 "movd   %1, %%mm0\n\t"
617                 "movd   4%1, %%mm3\n\t"
618                 "punpckldq 8%1, %%mm0\n\t"
619                 "punpckldq 12%1, %%mm3\n\t"
620                 "movq   %%mm0, %%mm1\n\t"
621                 "movq   %%mm0, %%mm2\n\t"
622                 "movq   %%mm3, %%mm4\n\t"
623                 "movq   %%mm3, %%mm5\n\t"
624                 "psllq  $7, %%mm0\n\t"
625                 "psllq  $7, %%mm3\n\t"
626                 "pand   %%mm7, %%mm0\n\t"
627                 "pand   %%mm7, %%mm3\n\t"
628                 "psrlq  $6, %%mm1\n\t"
629                 "psrlq  $6, %%mm4\n\t"
630                 "pand   %%mm6, %%mm1\n\t"
631                 "pand   %%mm6, %%mm4\n\t"
632                 "psrlq  $19, %%mm2\n\t"
633                 "psrlq  $19, %%mm5\n\t"
634                 "pand   %2, %%mm2\n\t"
635                 "pand   %2, %%mm5\n\t"
636                 "por    %%mm1, %%mm0\n\t"
637                 "por    %%mm4, %%mm3\n\t"
638                 "por    %%mm2, %%mm0\n\t"
639                 "por    %%mm5, %%mm3\n\t"
640                 "psllq  $16, %%mm3\n\t"
641                 "por    %%mm3, %%mm0\n\t"
642                 MOVNTQ" %%mm0, %0\n\t"
643                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
644                 d += 4;
645                 s += 16;
646         }
647         __asm __volatile(SFENCE:::"memory");
648         __asm __volatile(EMMS:::"memory");
649 #endif
650         while(s < end)
651         {
652                 register int rgb = *(uint32_t*)s; s += 4;
653                 *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
654         }
655 }
656
657 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, long src_size)
658 {
659         const uint8_t *s = src;
660         const uint8_t *end;
661 #ifdef HAVE_MMX
662         const uint8_t *mm_end;
663 #endif
664         uint16_t *d = (uint16_t *)dst;
665         end = s + src_size;
666 #ifdef HAVE_MMX
667         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
668         __asm __volatile(
669             "movq       %0, %%mm7\n\t"
670             "movq       %1, %%mm6\n\t"
671             ::"m"(red_16mask),"m"(green_16mask));
672         mm_end = end - 11;
673         while(s < mm_end)
674         {
675             __asm __volatile(
676                 PREFETCH" 32%1\n\t"
677                 "movd   %1, %%mm0\n\t"
678                 "movd   3%1, %%mm3\n\t"
679                 "punpckldq 6%1, %%mm0\n\t"
680                 "punpckldq 9%1, %%mm3\n\t"
681                 "movq   %%mm0, %%mm1\n\t"
682                 "movq   %%mm0, %%mm2\n\t"
683                 "movq   %%mm3, %%mm4\n\t"
684                 "movq   %%mm3, %%mm5\n\t"
685                 "psrlq  $3, %%mm0\n\t"
686                 "psrlq  $3, %%mm3\n\t"
687                 "pand   %2, %%mm0\n\t"
688                 "pand   %2, %%mm3\n\t"
689                 "psrlq  $5, %%mm1\n\t"
690                 "psrlq  $5, %%mm4\n\t"
691                 "pand   %%mm6, %%mm1\n\t"
692                 "pand   %%mm6, %%mm4\n\t"
693                 "psrlq  $8, %%mm2\n\t"
694                 "psrlq  $8, %%mm5\n\t"
695                 "pand   %%mm7, %%mm2\n\t"
696                 "pand   %%mm7, %%mm5\n\t"
697                 "por    %%mm1, %%mm0\n\t"
698                 "por    %%mm4, %%mm3\n\t"
699                 "por    %%mm2, %%mm0\n\t"
700                 "por    %%mm5, %%mm3\n\t"
701                 "psllq  $16, %%mm3\n\t"
702                 "por    %%mm3, %%mm0\n\t"
703                 MOVNTQ" %%mm0, %0\n\t"
704                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
705                 d += 4;
706                 s += 12;
707         }
708         __asm __volatile(SFENCE:::"memory");
709         __asm __volatile(EMMS:::"memory");
710 #endif
711         while(s < end)
712         {
713                 const int b= *s++;
714                 const int g= *s++;
715                 const int r= *s++;
716                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
717         }
718 }
719
720 static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
721 {
722         const uint8_t *s = src;
723         const uint8_t *end;
724 #ifdef HAVE_MMX
725         const uint8_t *mm_end;
726 #endif
727         uint16_t *d = (uint16_t *)dst;
728         end = s + src_size;
729 #ifdef HAVE_MMX
730         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
731         __asm __volatile(
732             "movq       %0, %%mm7\n\t"
733             "movq       %1, %%mm6\n\t"
734             ::"m"(red_16mask),"m"(green_16mask));
735         mm_end = end - 15;
736         while(s < mm_end)
737         {
738             __asm __volatile(
739                 PREFETCH" 32%1\n\t"
740                 "movd   %1, %%mm0\n\t"
741                 "movd   3%1, %%mm3\n\t"
742                 "punpckldq 6%1, %%mm0\n\t"
743                 "punpckldq 9%1, %%mm3\n\t"
744                 "movq   %%mm0, %%mm1\n\t"
745                 "movq   %%mm0, %%mm2\n\t"
746                 "movq   %%mm3, %%mm4\n\t"
747                 "movq   %%mm3, %%mm5\n\t"
748                 "psllq  $8, %%mm0\n\t"
749                 "psllq  $8, %%mm3\n\t"
750                 "pand   %%mm7, %%mm0\n\t"
751                 "pand   %%mm7, %%mm3\n\t"
752                 "psrlq  $5, %%mm1\n\t"
753                 "psrlq  $5, %%mm4\n\t"
754                 "pand   %%mm6, %%mm1\n\t"
755                 "pand   %%mm6, %%mm4\n\t"
756                 "psrlq  $19, %%mm2\n\t"
757                 "psrlq  $19, %%mm5\n\t"
758                 "pand   %2, %%mm2\n\t"
759                 "pand   %2, %%mm5\n\t"
760                 "por    %%mm1, %%mm0\n\t"
761                 "por    %%mm4, %%mm3\n\t"
762                 "por    %%mm2, %%mm0\n\t"
763                 "por    %%mm5, %%mm3\n\t"
764                 "psllq  $16, %%mm3\n\t"
765                 "por    %%mm3, %%mm0\n\t"
766                 MOVNTQ" %%mm0, %0\n\t"
767                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
768                 d += 4;
769                 s += 12;
770         }
771         __asm __volatile(SFENCE:::"memory");
772         __asm __volatile(EMMS:::"memory");
773 #endif
774         while(s < end)
775         {
776                 const int r= *s++;
777                 const int g= *s++;
778                 const int b= *s++;
779                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
780         }
781 }
782
783 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, long src_size)
784 {
785         const uint8_t *s = src;
786         const uint8_t *end;
787 #ifdef HAVE_MMX
788         const uint8_t *mm_end;
789 #endif
790         uint16_t *d = (uint16_t *)dst;
791         end = s + src_size;
792 #ifdef HAVE_MMX
793         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
794         __asm __volatile(
795             "movq       %0, %%mm7\n\t"
796             "movq       %1, %%mm6\n\t"
797             ::"m"(red_15mask),"m"(green_15mask));
798         mm_end = end - 11;
799         while(s < mm_end)
800         {
801             __asm __volatile(
802                 PREFETCH" 32%1\n\t"
803                 "movd   %1, %%mm0\n\t"
804                 "movd   3%1, %%mm3\n\t"
805                 "punpckldq 6%1, %%mm0\n\t"
806                 "punpckldq 9%1, %%mm3\n\t"
807                 "movq   %%mm0, %%mm1\n\t"
808                 "movq   %%mm0, %%mm2\n\t"
809                 "movq   %%mm3, %%mm4\n\t"
810                 "movq   %%mm3, %%mm5\n\t"
811                 "psrlq  $3, %%mm0\n\t"
812                 "psrlq  $3, %%mm3\n\t"
813                 "pand   %2, %%mm0\n\t"
814                 "pand   %2, %%mm3\n\t"
815                 "psrlq  $6, %%mm1\n\t"
816                 "psrlq  $6, %%mm4\n\t"
817                 "pand   %%mm6, %%mm1\n\t"
818                 "pand   %%mm6, %%mm4\n\t"
819                 "psrlq  $9, %%mm2\n\t"
820                 "psrlq  $9, %%mm5\n\t"
821                 "pand   %%mm7, %%mm2\n\t"
822                 "pand   %%mm7, %%mm5\n\t"
823                 "por    %%mm1, %%mm0\n\t"
824                 "por    %%mm4, %%mm3\n\t"
825                 "por    %%mm2, %%mm0\n\t"
826                 "por    %%mm5, %%mm3\n\t"
827                 "psllq  $16, %%mm3\n\t"
828                 "por    %%mm3, %%mm0\n\t"
829                 MOVNTQ" %%mm0, %0\n\t"
830                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
831                 d += 4;
832                 s += 12;
833         }
834         __asm __volatile(SFENCE:::"memory");
835         __asm __volatile(EMMS:::"memory");
836 #endif
837         while(s < end)
838         {
839                 const int b= *s++;
840                 const int g= *s++;
841                 const int r= *s++;
842                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
843         }
844 }
845
846 static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
847 {
848         const uint8_t *s = src;
849         const uint8_t *end;
850 #ifdef HAVE_MMX
851         const uint8_t *mm_end;
852 #endif
853         uint16_t *d = (uint16_t *)dst;
854         end = s + src_size;
855 #ifdef HAVE_MMX
856         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
857         __asm __volatile(
858             "movq       %0, %%mm7\n\t"
859             "movq       %1, %%mm6\n\t"
860             ::"m"(red_15mask),"m"(green_15mask));
861         mm_end = end - 15;
862         while(s < mm_end)
863         {
864             __asm __volatile(
865                 PREFETCH" 32%1\n\t"
866                 "movd   %1, %%mm0\n\t"
867                 "movd   3%1, %%mm3\n\t"
868                 "punpckldq 6%1, %%mm0\n\t"
869                 "punpckldq 9%1, %%mm3\n\t"
870                 "movq   %%mm0, %%mm1\n\t"
871                 "movq   %%mm0, %%mm2\n\t"
872                 "movq   %%mm3, %%mm4\n\t"
873                 "movq   %%mm3, %%mm5\n\t"
874                 "psllq  $7, %%mm0\n\t"
875                 "psllq  $7, %%mm3\n\t"
876                 "pand   %%mm7, %%mm0\n\t"
877                 "pand   %%mm7, %%mm3\n\t"
878                 "psrlq  $6, %%mm1\n\t"
879                 "psrlq  $6, %%mm4\n\t"
880                 "pand   %%mm6, %%mm1\n\t"
881                 "pand   %%mm6, %%mm4\n\t"
882                 "psrlq  $19, %%mm2\n\t"
883                 "psrlq  $19, %%mm5\n\t"
884                 "pand   %2, %%mm2\n\t"
885                 "pand   %2, %%mm5\n\t"
886                 "por    %%mm1, %%mm0\n\t"
887                 "por    %%mm4, %%mm3\n\t"
888                 "por    %%mm2, %%mm0\n\t"
889                 "por    %%mm5, %%mm3\n\t"
890                 "psllq  $16, %%mm3\n\t"
891                 "por    %%mm3, %%mm0\n\t"
892                 MOVNTQ" %%mm0, %0\n\t"
893                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
894                 d += 4;
895                 s += 12;
896         }
897         __asm __volatile(SFENCE:::"memory");
898         __asm __volatile(EMMS:::"memory");
899 #endif
900         while(s < end)
901         {
902                 const int r= *s++;
903                 const int g= *s++;
904                 const int b= *s++;
905                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
906         }
907 }
908
909 /*
910   I use here less accurate approximation by simply
911  left-shifting the input
912   value and filling the low order bits with
913  zeroes. This method improves png's
914   compression but this scheme cannot reproduce white exactly, since it does not
915   generate an all-ones maximum value; the net effect is to darken the
916   image slightly.
917
918   The better method should be "left bit replication":
919
920    4 3 2 1 0
921    ---------
922    1 1 0 1 1
923
924    7 6 5 4 3  2 1 0
925    ----------------
926    1 1 0 1 1  1 1 0
927    |=======|  |===|
928        |      Leftmost Bits Repeated to Fill Open Bits
929        |
930    Original Bits
931 */
932 static inline void RENAME(rgb15to24)(const uint8_t *src, uint8_t *dst, long src_size)
933 {
934         const uint16_t *end;
935 #ifdef HAVE_MMX
936         const uint16_t *mm_end;
937 #endif
938         uint8_t *d = (uint8_t *)dst;
939         const uint16_t *s = (uint16_t *)src;
940         end = s + src_size/2;
941 #ifdef HAVE_MMX
942         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
943         mm_end = end - 7;
944         while(s < mm_end)
945         {
946             __asm __volatile(
947                 PREFETCH" 32%1\n\t"
948                 "movq   %1, %%mm0\n\t"
949                 "movq   %1, %%mm1\n\t"
950                 "movq   %1, %%mm2\n\t"
951                 "pand   %2, %%mm0\n\t"
952                 "pand   %3, %%mm1\n\t"
953                 "pand   %4, %%mm2\n\t"
954                 "psllq  $3, %%mm0\n\t"
955                 "psrlq  $2, %%mm1\n\t"
956                 "psrlq  $7, %%mm2\n\t"
957                 "movq   %%mm0, %%mm3\n\t"
958                 "movq   %%mm1, %%mm4\n\t"
959                 "movq   %%mm2, %%mm5\n\t"
960                 "punpcklwd %5, %%mm0\n\t"
961                 "punpcklwd %5, %%mm1\n\t"
962                 "punpcklwd %5, %%mm2\n\t"
963                 "punpckhwd %5, %%mm3\n\t"
964                 "punpckhwd %5, %%mm4\n\t"
965                 "punpckhwd %5, %%mm5\n\t"
966                 "psllq  $8, %%mm1\n\t"
967                 "psllq  $16, %%mm2\n\t"
968                 "por    %%mm1, %%mm0\n\t"
969                 "por    %%mm2, %%mm0\n\t"
970                 "psllq  $8, %%mm4\n\t"
971                 "psllq  $16, %%mm5\n\t"
972                 "por    %%mm4, %%mm3\n\t"
973                 "por    %%mm5, %%mm3\n\t"
974
975                 "movq   %%mm0, %%mm6\n\t"
976                 "movq   %%mm3, %%mm7\n\t"
977                 
978                 "movq   8%1, %%mm0\n\t"
979                 "movq   8%1, %%mm1\n\t"
980                 "movq   8%1, %%mm2\n\t"
981                 "pand   %2, %%mm0\n\t"
982                 "pand   %3, %%mm1\n\t"
983                 "pand   %4, %%mm2\n\t"
984                 "psllq  $3, %%mm0\n\t"
985                 "psrlq  $2, %%mm1\n\t"
986                 "psrlq  $7, %%mm2\n\t"
987                 "movq   %%mm0, %%mm3\n\t"
988                 "movq   %%mm1, %%mm4\n\t"
989                 "movq   %%mm2, %%mm5\n\t"
990                 "punpcklwd %5, %%mm0\n\t"
991                 "punpcklwd %5, %%mm1\n\t"
992                 "punpcklwd %5, %%mm2\n\t"
993                 "punpckhwd %5, %%mm3\n\t"
994                 "punpckhwd %5, %%mm4\n\t"
995                 "punpckhwd %5, %%mm5\n\t"
996                 "psllq  $8, %%mm1\n\t"
997                 "psllq  $16, %%mm2\n\t"
998                 "por    %%mm1, %%mm0\n\t"
999                 "por    %%mm2, %%mm0\n\t"
1000                 "psllq  $8, %%mm4\n\t"
1001                 "psllq  $16, %%mm5\n\t"
1002                 "por    %%mm4, %%mm3\n\t"
1003                 "por    %%mm5, %%mm3\n\t"
1004
1005                 :"=m"(*d)
1006                 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)
1007                 :"memory");
1008             /* Borrowed 32 to 24 */
1009             __asm __volatile(
1010                 "movq   %%mm0, %%mm4\n\t"
1011                 "movq   %%mm3, %%mm5\n\t"
1012                 "movq   %%mm6, %%mm0\n\t"
1013                 "movq   %%mm7, %%mm1\n\t"
1014                 
1015                 "movq   %%mm4, %%mm6\n\t"
1016                 "movq   %%mm5, %%mm7\n\t"
1017                 "movq   %%mm0, %%mm2\n\t"
1018                 "movq   %%mm1, %%mm3\n\t"
1019
1020                 "psrlq  $8, %%mm2\n\t"
1021                 "psrlq  $8, %%mm3\n\t"
1022                 "psrlq  $8, %%mm6\n\t"
1023                 "psrlq  $8, %%mm7\n\t"
1024                 "pand   %2, %%mm0\n\t"
1025                 "pand   %2, %%mm1\n\t"
1026                 "pand   %2, %%mm4\n\t"
1027                 "pand   %2, %%mm5\n\t"
1028                 "pand   %3, %%mm2\n\t"
1029                 "pand   %3, %%mm3\n\t"
1030                 "pand   %3, %%mm6\n\t"
1031                 "pand   %3, %%mm7\n\t"
1032                 "por    %%mm2, %%mm0\n\t"
1033                 "por    %%mm3, %%mm1\n\t"
1034                 "por    %%mm6, %%mm4\n\t"
1035                 "por    %%mm7, %%mm5\n\t"
1036
1037                 "movq   %%mm1, %%mm2\n\t"
1038                 "movq   %%mm4, %%mm3\n\t"
1039                 "psllq  $48, %%mm2\n\t"
1040                 "psllq  $32, %%mm3\n\t"
1041                 "pand   %4, %%mm2\n\t"
1042                 "pand   %5, %%mm3\n\t"
1043                 "por    %%mm2, %%mm0\n\t"
1044                 "psrlq  $16, %%mm1\n\t"
1045                 "psrlq  $32, %%mm4\n\t"
1046                 "psllq  $16, %%mm5\n\t"
1047                 "por    %%mm3, %%mm1\n\t"
1048                 "pand   %6, %%mm5\n\t"
1049                 "por    %%mm5, %%mm4\n\t"
1050
1051                 MOVNTQ" %%mm0, %0\n\t"
1052                 MOVNTQ" %%mm1, 8%0\n\t"
1053                 MOVNTQ" %%mm4, 16%0"
1054
1055                 :"=m"(*d)
1056                 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1057                 :"memory");
1058                 d += 24;
1059                 s += 8;
1060         }
1061         __asm __volatile(SFENCE:::"memory");
1062         __asm __volatile(EMMS:::"memory");
1063 #endif
1064         while(s < end)
1065         {
1066                 register uint16_t bgr;
1067                 bgr = *s++;
1068                 *d++ = (bgr&0x1F)<<3;
1069                 *d++ = (bgr&0x3E0)>>2;
1070                 *d++ = (bgr&0x7C00)>>7;
1071         }
1072 }
1073
1074 static inline void RENAME(rgb16to24)(const uint8_t *src, uint8_t *dst, long src_size)
1075 {
1076         const uint16_t *end;
1077 #ifdef HAVE_MMX
1078         const uint16_t *mm_end;
1079 #endif
1080         uint8_t *d = (uint8_t *)dst;
1081         const uint16_t *s = (const uint16_t *)src;
1082         end = s + src_size/2;
1083 #ifdef HAVE_MMX
1084         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1085         mm_end = end - 7;
1086         while(s < mm_end)
1087         {
1088             __asm __volatile(
1089                 PREFETCH" 32%1\n\t"
1090                 "movq   %1, %%mm0\n\t"
1091                 "movq   %1, %%mm1\n\t"
1092                 "movq   %1, %%mm2\n\t"
1093                 "pand   %2, %%mm0\n\t"
1094                 "pand   %3, %%mm1\n\t"
1095                 "pand   %4, %%mm2\n\t"
1096                 "psllq  $3, %%mm0\n\t"
1097                 "psrlq  $3, %%mm1\n\t"
1098                 "psrlq  $8, %%mm2\n\t"
1099                 "movq   %%mm0, %%mm3\n\t"
1100                 "movq   %%mm1, %%mm4\n\t"
1101                 "movq   %%mm2, %%mm5\n\t"
1102                 "punpcklwd %5, %%mm0\n\t"
1103                 "punpcklwd %5, %%mm1\n\t"
1104                 "punpcklwd %5, %%mm2\n\t"
1105                 "punpckhwd %5, %%mm3\n\t"
1106                 "punpckhwd %5, %%mm4\n\t"
1107                 "punpckhwd %5, %%mm5\n\t"
1108                 "psllq  $8, %%mm1\n\t"
1109                 "psllq  $16, %%mm2\n\t"
1110                 "por    %%mm1, %%mm0\n\t"
1111                 "por    %%mm2, %%mm0\n\t"
1112                 "psllq  $8, %%mm4\n\t"
1113                 "psllq  $16, %%mm5\n\t"
1114                 "por    %%mm4, %%mm3\n\t"
1115                 "por    %%mm5, %%mm3\n\t"
1116                 
1117                 "movq   %%mm0, %%mm6\n\t"
1118                 "movq   %%mm3, %%mm7\n\t"
1119
1120                 "movq   8%1, %%mm0\n\t"
1121                 "movq   8%1, %%mm1\n\t"
1122                 "movq   8%1, %%mm2\n\t"
1123                 "pand   %2, %%mm0\n\t"
1124                 "pand   %3, %%mm1\n\t"
1125                 "pand   %4, %%mm2\n\t"
1126                 "psllq  $3, %%mm0\n\t"
1127                 "psrlq  $3, %%mm1\n\t"
1128                 "psrlq  $8, %%mm2\n\t"
1129                 "movq   %%mm0, %%mm3\n\t"
1130                 "movq   %%mm1, %%mm4\n\t"
1131                 "movq   %%mm2, %%mm5\n\t"
1132                 "punpcklwd %5, %%mm0\n\t"
1133                 "punpcklwd %5, %%mm1\n\t"
1134                 "punpcklwd %5, %%mm2\n\t"
1135                 "punpckhwd %5, %%mm3\n\t"
1136                 "punpckhwd %5, %%mm4\n\t"
1137                 "punpckhwd %5, %%mm5\n\t"
1138                 "psllq  $8, %%mm1\n\t"
1139                 "psllq  $16, %%mm2\n\t"
1140                 "por    %%mm1, %%mm0\n\t"
1141                 "por    %%mm2, %%mm0\n\t"
1142                 "psllq  $8, %%mm4\n\t"
1143                 "psllq  $16, %%mm5\n\t"
1144                 "por    %%mm4, %%mm3\n\t"
1145                 "por    %%mm5, %%mm3\n\t"
1146                 :"=m"(*d)
1147                 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)           
1148                 :"memory");
1149             /* Borrowed 32 to 24 */
1150             __asm __volatile(
1151                 "movq   %%mm0, %%mm4\n\t"
1152                 "movq   %%mm3, %%mm5\n\t"
1153                 "movq   %%mm6, %%mm0\n\t"
1154                 "movq   %%mm7, %%mm1\n\t"
1155                 
1156                 "movq   %%mm4, %%mm6\n\t"
1157                 "movq   %%mm5, %%mm7\n\t"
1158                 "movq   %%mm0, %%mm2\n\t"
1159                 "movq   %%mm1, %%mm3\n\t"
1160
1161                 "psrlq  $8, %%mm2\n\t"
1162                 "psrlq  $8, %%mm3\n\t"
1163                 "psrlq  $8, %%mm6\n\t"
1164                 "psrlq  $8, %%mm7\n\t"
1165                 "pand   %2, %%mm0\n\t"
1166                 "pand   %2, %%mm1\n\t"
1167                 "pand   %2, %%mm4\n\t"
1168                 "pand   %2, %%mm5\n\t"
1169                 "pand   %3, %%mm2\n\t"
1170                 "pand   %3, %%mm3\n\t"
1171                 "pand   %3, %%mm6\n\t"
1172                 "pand   %3, %%mm7\n\t"
1173                 "por    %%mm2, %%mm0\n\t"
1174                 "por    %%mm3, %%mm1\n\t"
1175                 "por    %%mm6, %%mm4\n\t"
1176                 "por    %%mm7, %%mm5\n\t"
1177
1178                 "movq   %%mm1, %%mm2\n\t"
1179                 "movq   %%mm4, %%mm3\n\t"
1180                 "psllq  $48, %%mm2\n\t"
1181                 "psllq  $32, %%mm3\n\t"
1182                 "pand   %4, %%mm2\n\t"
1183                 "pand   %5, %%mm3\n\t"
1184                 "por    %%mm2, %%mm0\n\t"
1185                 "psrlq  $16, %%mm1\n\t"
1186                 "psrlq  $32, %%mm4\n\t"
1187                 "psllq  $16, %%mm5\n\t"
1188                 "por    %%mm3, %%mm1\n\t"
1189                 "pand   %6, %%mm5\n\t"
1190                 "por    %%mm5, %%mm4\n\t"
1191
1192                 MOVNTQ" %%mm0, %0\n\t"
1193                 MOVNTQ" %%mm1, 8%0\n\t"
1194                 MOVNTQ" %%mm4, 16%0"
1195
1196                 :"=m"(*d)
1197                 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1198                 :"memory");
1199                 d += 24;
1200                 s += 8;
1201         }
1202         __asm __volatile(SFENCE:::"memory");
1203         __asm __volatile(EMMS:::"memory");
1204 #endif
1205         while(s < end)
1206         {
1207                 register uint16_t bgr;
1208                 bgr = *s++;
1209                 *d++ = (bgr&0x1F)<<3;
1210                 *d++ = (bgr&0x7E0)>>3;
1211                 *d++ = (bgr&0xF800)>>8;
1212         }
1213 }
1214
1215 static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_size)
1216 {
1217         const uint16_t *end;
1218 #ifdef HAVE_MMX
1219         const uint16_t *mm_end;
1220 #endif
1221         uint8_t *d = (uint8_t *)dst;
1222         const uint16_t *s = (const uint16_t *)src;
1223         end = s + src_size/2;
1224 #ifdef HAVE_MMX
1225         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1226         __asm __volatile("pxor  %%mm7,%%mm7\n\t":::"memory");
1227         mm_end = end - 3;
1228         while(s < mm_end)
1229         {
1230             __asm __volatile(
1231                 PREFETCH" 32%1\n\t"
1232                 "movq   %1, %%mm0\n\t"
1233                 "movq   %1, %%mm1\n\t"
1234                 "movq   %1, %%mm2\n\t"
1235                 "pand   %2, %%mm0\n\t"
1236                 "pand   %3, %%mm1\n\t"
1237                 "pand   %4, %%mm2\n\t"
1238                 "psllq  $3, %%mm0\n\t"
1239                 "psrlq  $2, %%mm1\n\t"
1240                 "psrlq  $7, %%mm2\n\t"
1241                 "movq   %%mm0, %%mm3\n\t"
1242                 "movq   %%mm1, %%mm4\n\t"
1243                 "movq   %%mm2, %%mm5\n\t"
1244                 "punpcklwd %%mm7, %%mm0\n\t"
1245                 "punpcklwd %%mm7, %%mm1\n\t"
1246                 "punpcklwd %%mm7, %%mm2\n\t"
1247                 "punpckhwd %%mm7, %%mm3\n\t"
1248                 "punpckhwd %%mm7, %%mm4\n\t"
1249                 "punpckhwd %%mm7, %%mm5\n\t"
1250                 "psllq  $8, %%mm1\n\t"
1251                 "psllq  $16, %%mm2\n\t"
1252                 "por    %%mm1, %%mm0\n\t"
1253                 "por    %%mm2, %%mm0\n\t"
1254                 "psllq  $8, %%mm4\n\t"
1255                 "psllq  $16, %%mm5\n\t"
1256                 "por    %%mm4, %%mm3\n\t"
1257                 "por    %%mm5, %%mm3\n\t"
1258                 MOVNTQ" %%mm0, %0\n\t"
1259                 MOVNTQ" %%mm3, 8%0\n\t"
1260                 :"=m"(*d)
1261                 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r)
1262                 :"memory");
1263                 d += 16;
1264                 s += 4;
1265         }
1266         __asm __volatile(SFENCE:::"memory");
1267         __asm __volatile(EMMS:::"memory");
1268 #endif
1269         while(s < end)
1270         {
1271 #if 0 //slightly slower on athlon
1272                 int bgr= *s++;
1273                 *((uint32_t*)d)++ = ((bgr&0x1F)<<3) + ((bgr&0x3E0)<<6) + ((bgr&0x7C00)<<9);
1274 #else
1275                 register uint16_t bgr;
1276                 bgr = *s++;
1277 #ifdef WORDS_BIGENDIAN
1278                 *d++ = 0;
1279                 *d++ = (bgr&0x7C00)>>7;
1280                 *d++ = (bgr&0x3E0)>>2;
1281                 *d++ = (bgr&0x1F)<<3;
1282 #else
1283                 *d++ = (bgr&0x1F)<<3;
1284                 *d++ = (bgr&0x3E0)>>2;
1285                 *d++ = (bgr&0x7C00)>>7;
1286                 *d++ = 0;
1287 #endif
1288
1289 #endif
1290         }
1291 }
1292
1293 static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_size)
1294 {
1295         const uint16_t *end;
1296 #ifdef HAVE_MMX
1297         const uint16_t *mm_end;
1298 #endif
1299         uint8_t *d = (uint8_t *)dst;
1300         const uint16_t *s = (uint16_t *)src;
1301         end = s + src_size/2;
1302 #ifdef HAVE_MMX
1303         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1304         __asm __volatile("pxor  %%mm7,%%mm7\n\t":::"memory");
1305         mm_end = end - 3;
1306         while(s < mm_end)
1307         {
1308             __asm __volatile(
1309                 PREFETCH" 32%1\n\t"
1310                 "movq   %1, %%mm0\n\t"
1311                 "movq   %1, %%mm1\n\t"
1312                 "movq   %1, %%mm2\n\t"
1313                 "pand   %2, %%mm0\n\t"
1314                 "pand   %3, %%mm1\n\t"
1315                 "pand   %4, %%mm2\n\t"
1316                 "psllq  $3, %%mm0\n\t"
1317                 "psrlq  $3, %%mm1\n\t"
1318                 "psrlq  $8, %%mm2\n\t"
1319                 "movq   %%mm0, %%mm3\n\t"
1320                 "movq   %%mm1, %%mm4\n\t"
1321                 "movq   %%mm2, %%mm5\n\t"
1322                 "punpcklwd %%mm7, %%mm0\n\t"
1323                 "punpcklwd %%mm7, %%mm1\n\t"
1324                 "punpcklwd %%mm7, %%mm2\n\t"
1325                 "punpckhwd %%mm7, %%mm3\n\t"
1326                 "punpckhwd %%mm7, %%mm4\n\t"
1327                 "punpckhwd %%mm7, %%mm5\n\t"
1328                 "psllq  $8, %%mm1\n\t"
1329                 "psllq  $16, %%mm2\n\t"
1330                 "por    %%mm1, %%mm0\n\t"
1331                 "por    %%mm2, %%mm0\n\t"
1332                 "psllq  $8, %%mm4\n\t"
1333                 "psllq  $16, %%mm5\n\t"
1334                 "por    %%mm4, %%mm3\n\t"
1335                 "por    %%mm5, %%mm3\n\t"
1336                 MOVNTQ" %%mm0, %0\n\t"
1337                 MOVNTQ" %%mm3, 8%0\n\t"
1338                 :"=m"(*d)
1339                 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r)
1340                 :"memory");
1341                 d += 16;
1342                 s += 4;
1343         }
1344         __asm __volatile(SFENCE:::"memory");
1345         __asm __volatile(EMMS:::"memory");
1346 #endif
1347         while(s < end)
1348         {
1349                 register uint16_t bgr;
1350                 bgr = *s++;
1351 #ifdef WORDS_BIGENDIAN
1352                 *d++ = 0;
1353                 *d++ = (bgr&0xF800)>>8;
1354                 *d++ = (bgr&0x7E0)>>3;
1355                 *d++ = (bgr&0x1F)<<3;
1356 #else
1357                 *d++ = (bgr&0x1F)<<3;
1358                 *d++ = (bgr&0x7E0)>>3;
1359                 *d++ = (bgr&0xF800)>>8;
1360                 *d++ = 0;
1361 #endif
1362         }
1363 }
1364
1365 static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, long src_size)
1366 {
1367     uint8_t *d = dst, *s = (uint8_t *) src;
1368     const uint8_t *end = s + src_size;
1369 #ifdef HAVE_MMX
1370         __asm __volatile(
1371                 "       "PREFETCH" (%1)                 \n"
1372                 "       movq %3, %%mm7                  \n"
1373                 "       pxor %4, %%mm7                  \n"
1374                 "       movq %%mm7, %%mm6               \n"
1375                 "       pxor %5, %%mm7                  \n"
1376                 "       jmp 2f                          \n"
1377                         ASMALIGN(4)
1378                 "1:                                     \n"
1379                 "       "PREFETCH" 32(%1)               \n"
1380                 "       movq (%1), %%mm0                \n"
1381                 "       movq 8(%1), %%mm1               \n"
1382 # ifdef HAVE_MMX2
1383                 "       pshufw $177, %%mm0, %%mm3       \n"
1384                 "       pshufw $177, %%mm1, %%mm5       \n"
1385                 "       pand %%mm7, %%mm0               \n"
1386                 "       pand %%mm6, %%mm3               \n"
1387                 "       pand %%mm7, %%mm1               \n"
1388                 "       pand %%mm6, %%mm5               \n"
1389                 "       por %%mm3, %%mm0                \n"
1390                 "       por %%mm5, %%mm1                \n"
1391 # else
1392                 "       movq %%mm0, %%mm2               \n"
1393                 "       movq %%mm1, %%mm4               \n"
1394                 "       pand %%mm7, %%mm0               \n"
1395                 "       pand %%mm6, %%mm2               \n"
1396                 "       pand %%mm7, %%mm1               \n"
1397                 "       pand %%mm6, %%mm4               \n"
1398                 "       movq %%mm2, %%mm3               \n"
1399                 "       movq %%mm4, %%mm5               \n"
1400                 "       pslld $16, %%mm2                \n"
1401                 "       psrld $16, %%mm3                \n"
1402                 "       pslld $16, %%mm4                \n"
1403                 "       psrld $16, %%mm5                \n"
1404                 "       por %%mm2, %%mm0                \n"
1405                 "       por %%mm4, %%mm1                \n"
1406                 "       por %%mm3, %%mm0                \n"
1407                 "       por %%mm5, %%mm1                \n"
1408 # endif
1409                 "       "MOVNTQ" %%mm0, (%0)            \n"
1410                 "       "MOVNTQ" %%mm1, 8(%0)           \n"
1411                 "       add $16, %0                     \n"
1412                 "       add $16, %1                     \n"
1413                 "2:                                     \n"
1414                 "       cmp %1, %2                      \n"
1415                 "       ja 1b                           \n"
1416                 "       "SFENCE"                        \n"
1417                 "       "EMMS"                          \n"
1418                 : "+r"(d), "+r"(s)
1419                 : "r" (end-15), "m" (mask32b), "m" (mask32r), "m" (mmx_one)
1420                 : "memory");
1421 #endif
1422         for (; s<end; s+=4, d+=4) {
1423                 int v = *(uint32_t *)s, g = v & 0xff00;
1424                 v &= 0xff00ff;
1425                 *(uint32_t *)d = (v>>16) + g + (v<<16);
1426         }
1427 }
1428
1429 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, long src_size)
1430 {
1431         unsigned i;
1432 #ifdef HAVE_MMX
1433         long mmx_size= 23 - src_size;
1434         asm volatile (
1435                 "movq "MANGLE(mask24r)", %%mm5  \n\t"
1436                 "movq "MANGLE(mask24g)", %%mm6  \n\t"
1437                 "movq "MANGLE(mask24b)", %%mm7  \n\t"
1438                 ASMALIGN(4)
1439                 "1:                             \n\t"
1440                 PREFETCH" 32(%1, %%"REG_a")     \n\t"
1441                 "movq   (%1, %%"REG_a"), %%mm0  \n\t" // BGR BGR BG
1442                 "movq   (%1, %%"REG_a"), %%mm1  \n\t" // BGR BGR BG
1443                 "movq  2(%1, %%"REG_a"), %%mm2  \n\t" // R BGR BGR B
1444                 "psllq $16, %%mm0               \n\t" // 00 BGR BGR
1445                 "pand %%mm5, %%mm0              \n\t"
1446                 "pand %%mm6, %%mm1              \n\t"
1447                 "pand %%mm7, %%mm2              \n\t"
1448                 "por %%mm0, %%mm1               \n\t"
1449                 "por %%mm2, %%mm1               \n\t"                
1450                 "movq  6(%1, %%"REG_a"), %%mm0  \n\t" // BGR BGR BG
1451                 MOVNTQ" %%mm1,   (%2, %%"REG_a")\n\t" // RGB RGB RG
1452                 "movq  8(%1, %%"REG_a"), %%mm1  \n\t" // R BGR BGR B
1453                 "movq 10(%1, %%"REG_a"), %%mm2  \n\t" // GR BGR BGR
1454                 "pand %%mm7, %%mm0              \n\t"
1455                 "pand %%mm5, %%mm1              \n\t"
1456                 "pand %%mm6, %%mm2              \n\t"
1457                 "por %%mm0, %%mm1               \n\t"
1458                 "por %%mm2, %%mm1               \n\t"                
1459                 "movq 14(%1, %%"REG_a"), %%mm0  \n\t" // R BGR BGR B
1460                 MOVNTQ" %%mm1,  8(%2, %%"REG_a")\n\t" // B RGB RGB R
1461                 "movq 16(%1, %%"REG_a"), %%mm1  \n\t" // GR BGR BGR
1462                 "movq 18(%1, %%"REG_a"), %%mm2  \n\t" // BGR BGR BG
1463                 "pand %%mm6, %%mm0              \n\t"
1464                 "pand %%mm7, %%mm1              \n\t"
1465                 "pand %%mm5, %%mm2              \n\t"
1466                 "por %%mm0, %%mm1               \n\t"
1467                 "por %%mm2, %%mm1               \n\t"                
1468                 MOVNTQ" %%mm1, 16(%2, %%"REG_a")\n\t"
1469                 "add $24, %%"REG_a"             \n\t"
1470                 " js 1b                         \n\t"
1471                 : "+a" (mmx_size)
1472                 : "r" (src-mmx_size), "r"(dst-mmx_size)
1473         );
1474
1475         __asm __volatile(SFENCE:::"memory");
1476         __asm __volatile(EMMS:::"memory");
1477
1478         if(mmx_size==23) return; //finihsed, was multiple of 8
1479
1480         src+= src_size;
1481         dst+= src_size;
1482         src_size= 23-mmx_size;
1483         src-= src_size;
1484         dst-= src_size;
1485 #endif
1486         for(i=0; i<src_size; i+=3)
1487         {
1488                 register uint8_t x;
1489                 x          = src[i + 2];
1490                 dst[i + 1] = src[i + 1];
1491                 dst[i + 2] = src[i + 0];
1492                 dst[i + 0] = x;
1493         }
1494 }
1495
1496 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1497         long width, long height,
1498         long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1499 {
1500         long y;
1501         const long chromWidth= width>>1;
1502         for(y=0; y<height; y++)
1503         {
1504 #ifdef HAVE_MMX
1505 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1506                 asm volatile(
1507                         "xor %%"REG_a", %%"REG_a"       \n\t"
1508                         ASMALIGN(4)
1509                         "1:                             \n\t"
1510                         PREFETCH" 32(%1, %%"REG_a", 2)  \n\t"
1511                         PREFETCH" 32(%2, %%"REG_a")     \n\t"
1512                         PREFETCH" 32(%3, %%"REG_a")     \n\t"
1513                         "movq (%2, %%"REG_a"), %%mm0    \n\t" // U(0)
1514                         "movq %%mm0, %%mm2              \n\t" // U(0)
1515                         "movq (%3, %%"REG_a"), %%mm1    \n\t" // V(0)
1516                         "punpcklbw %%mm1, %%mm0         \n\t" // UVUV UVUV(0)
1517                         "punpckhbw %%mm1, %%mm2         \n\t" // UVUV UVUV(8)
1518
1519                         "movq (%1, %%"REG_a",2), %%mm3  \n\t" // Y(0)
1520                         "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1521                         "movq %%mm3, %%mm4              \n\t" // Y(0)
1522                         "movq %%mm5, %%mm6              \n\t" // Y(8)
1523                         "punpcklbw %%mm0, %%mm3         \n\t" // YUYV YUYV(0)
1524                         "punpckhbw %%mm0, %%mm4         \n\t" // YUYV YUYV(4)
1525                         "punpcklbw %%mm2, %%mm5         \n\t" // YUYV YUYV(8)
1526                         "punpckhbw %%mm2, %%mm6         \n\t" // YUYV YUYV(12)
1527
1528                         MOVNTQ" %%mm3, (%0, %%"REG_a", 4)\n\t"
1529                         MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1530                         MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4)\n\t"
1531                         MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1532
1533                         "add $8, %%"REG_a"              \n\t"
1534                         "cmp %4, %%"REG_a"              \n\t"
1535                         " jb 1b                         \n\t"
1536                         ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1537                         : "%"REG_a
1538                 );
1539 #else
1540
1541 #if defined ARCH_ALPHA && defined HAVE_MVI
1542 #define pl2yuy2(n)                                      \
1543         y1 = yc[n];                                     \
1544         y2 = yc2[n];                                    \
1545         u = uc[n];                                      \
1546         v = vc[n];                                      \
1547         asm("unpkbw %1, %0" : "=r"(y1) : "r"(y1));      \
1548         asm("unpkbw %1, %0" : "=r"(y2) : "r"(y2));      \
1549         asm("unpkbl %1, %0" : "=r"(u) : "r"(u));        \
1550         asm("unpkbl %1, %0" : "=r"(v) : "r"(v));        \
1551         yuv1 = (u << 8) + (v << 24);                    \
1552         yuv2 = yuv1 + y2;                               \
1553         yuv1 += y1;                                     \
1554         qdst[n] = yuv1;                                 \
1555         qdst2[n] = yuv2;
1556
1557                 int i;
1558                 uint64_t *qdst = (uint64_t *) dst;
1559                 uint64_t *qdst2 = (uint64_t *) (dst + dstStride);
1560                 const uint32_t *yc = (uint32_t *) ysrc;
1561                 const uint32_t *yc2 = (uint32_t *) (ysrc + lumStride);
1562                 const uint16_t *uc = (uint16_t*) usrc, *vc = (uint16_t*) vsrc;
1563                 for(i = 0; i < chromWidth; i += 8){
1564                         uint64_t y1, y2, yuv1, yuv2;
1565                         uint64_t u, v;
1566                         /* Prefetch */
1567                         asm("ldq $31,64(%0)" :: "r"(yc));
1568                         asm("ldq $31,64(%0)" :: "r"(yc2));
1569                         asm("ldq $31,64(%0)" :: "r"(uc));
1570                         asm("ldq $31,64(%0)" :: "r"(vc));
1571
1572                         pl2yuy2(0);
1573                         pl2yuy2(1);
1574                         pl2yuy2(2);
1575                         pl2yuy2(3);
1576
1577                         yc += 4;
1578                         yc2 += 4;
1579                         uc += 4;
1580                         vc += 4;
1581                         qdst += 4;
1582                         qdst2 += 4;
1583                 }
1584                 y++;
1585                 ysrc += lumStride;
1586                 dst += dstStride;
1587
1588 #elif __WORDSIZE >= 64
1589                 int i;
1590                 uint64_t *ldst = (uint64_t *) dst;
1591                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1592                 for(i = 0; i < chromWidth; i += 2){
1593                         uint64_t k, l;
1594                         k = yc[0] + (uc[0] << 8) +
1595                             (yc[1] << 16) + (vc[0] << 24);
1596                         l = yc[2] + (uc[1] << 8) +
1597                             (yc[3] << 16) + (vc[1] << 24);
1598                         *ldst++ = k + (l << 32);
1599                         yc += 4;
1600                         uc += 2;
1601                         vc += 2;
1602                 }
1603
1604 #else
1605                 int i, *idst = (int32_t *) dst;
1606                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1607                 for(i = 0; i < chromWidth; i++){
1608 #ifdef WORDS_BIGENDIAN
1609                         *idst++ = (yc[0] << 24)+ (uc[0] << 16) +
1610                             (yc[1] << 8) + (vc[0] << 0);
1611 #else
1612                         *idst++ = yc[0] + (uc[0] << 8) +
1613                             (yc[1] << 16) + (vc[0] << 24);
1614 #endif
1615                         yc += 2;
1616                         uc++;
1617                         vc++;
1618                 }
1619 #endif
1620 #endif
1621                 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1622                 {
1623                         usrc += chromStride;
1624                         vsrc += chromStride;
1625                 }
1626                 ysrc += lumStride;
1627                 dst += dstStride;
1628         }
1629 #ifdef HAVE_MMX
1630 asm(    EMMS" \n\t"
1631         SFENCE" \n\t"
1632         :::"memory");
1633 #endif
1634 }
1635
1636 /**
1637  *
1638  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1639  * problem for anyone then tell me, and ill fix it)
1640  */
1641 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1642         long width, long height,
1643         long lumStride, long chromStride, long dstStride)
1644 {
1645         //FIXME interpolate chroma
1646         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1647 }
1648
1649 static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1650         long width, long height,
1651         long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1652 {
1653         long y;
1654         const long chromWidth= width>>1;
1655         for(y=0; y<height; y++)
1656         {
1657 #ifdef HAVE_MMX
1658 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1659                 asm volatile(
1660                         "xor %%"REG_a", %%"REG_a"       \n\t"
1661                         ASMALIGN(4)
1662                         "1:                             \n\t"
1663                         PREFETCH" 32(%1, %%"REG_a", 2)  \n\t"
1664                         PREFETCH" 32(%2, %%"REG_a")     \n\t"
1665                         PREFETCH" 32(%3, %%"REG_a")     \n\t"
1666                         "movq (%2, %%"REG_a"), %%mm0    \n\t" // U(0)
1667                         "movq %%mm0, %%mm2              \n\t" // U(0)
1668                         "movq (%3, %%"REG_a"), %%mm1    \n\t" // V(0)
1669                         "punpcklbw %%mm1, %%mm0         \n\t" // UVUV UVUV(0)
1670                         "punpckhbw %%mm1, %%mm2         \n\t" // UVUV UVUV(8)
1671
1672                         "movq (%1, %%"REG_a",2), %%mm3  \n\t" // Y(0)
1673                         "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1674                         "movq %%mm0, %%mm4              \n\t" // Y(0)
1675                         "movq %%mm2, %%mm6              \n\t" // Y(8)
1676                         "punpcklbw %%mm3, %%mm0         \n\t" // YUYV YUYV(0)
1677                         "punpckhbw %%mm3, %%mm4         \n\t" // YUYV YUYV(4)
1678                         "punpcklbw %%mm5, %%mm2         \n\t" // YUYV YUYV(8)
1679                         "punpckhbw %%mm5, %%mm6         \n\t" // YUYV YUYV(12)
1680
1681                         MOVNTQ" %%mm0, (%0, %%"REG_a", 4)\n\t"
1682                         MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1683                         MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4)\n\t"
1684                         MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1685
1686                         "add $8, %%"REG_a"              \n\t"
1687                         "cmp %4, %%"REG_a"              \n\t"
1688                         " jb 1b                         \n\t"
1689                         ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1690                         : "%"REG_a
1691                 );
1692 #else
1693 //FIXME adapt the alpha asm code from yv12->yuy2
1694
1695 #if __WORDSIZE >= 64
1696                 int i;
1697                 uint64_t *ldst = (uint64_t *) dst;
1698                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1699                 for(i = 0; i < chromWidth; i += 2){
1700                         uint64_t k, l;
1701                         k = uc[0] + (yc[0] << 8) +
1702                             (vc[0] << 16) + (yc[1] << 24);
1703                         l = uc[1] + (yc[2] << 8) +
1704                             (vc[1] << 16) + (yc[3] << 24);
1705                         *ldst++ = k + (l << 32);
1706                         yc += 4;
1707                         uc += 2;
1708                         vc += 2;
1709                 }
1710
1711 #else
1712                 int i, *idst = (int32_t *) dst;
1713                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1714                 for(i = 0; i < chromWidth; i++){
1715 #ifdef WORDS_BIGENDIAN
1716                         *idst++ = (uc[0] << 24)+ (yc[0] << 16) +
1717                             (vc[0] << 8) + (yc[1] << 0);
1718 #else
1719                         *idst++ = uc[0] + (yc[0] << 8) +
1720                             (vc[0] << 16) + (yc[1] << 24);
1721 #endif
1722                         yc += 2;
1723                         uc++;
1724                         vc++;
1725                 }
1726 #endif
1727 #endif
1728                 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1729                 {
1730                         usrc += chromStride;
1731                         vsrc += chromStride;
1732                 }
1733                 ysrc += lumStride;
1734                 dst += dstStride;
1735         }
1736 #ifdef HAVE_MMX
1737 asm(    EMMS" \n\t"
1738         SFENCE" \n\t"
1739         :::"memory");
1740 #endif
1741 }
1742
1743 /**
1744  *
1745  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1746  * problem for anyone then tell me, and ill fix it)
1747  */
1748 static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1749         long width, long height,
1750         long lumStride, long chromStride, long dstStride)
1751 {
1752         //FIXME interpolate chroma
1753         RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1754 }
1755
1756 /**
1757  *
1758  * width should be a multiple of 16
1759  */
1760 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1761         long width, long height,
1762         long lumStride, long chromStride, long dstStride)
1763 {
1764         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1765 }
1766
1767 /**
1768  *
1769  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1770  * problem for anyone then tell me, and ill fix it)
1771  */
1772 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1773         long width, long height,
1774         long lumStride, long chromStride, long srcStride)
1775 {
1776         long y;
1777         const long chromWidth= width>>1;
1778         for(y=0; y<height; y+=2)
1779         {
1780 #ifdef HAVE_MMX
1781                 asm volatile(
1782                         "xor %%"REG_a", %%"REG_a"       \n\t"
1783                         "pcmpeqw %%mm7, %%mm7           \n\t"
1784                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
1785                         ASMALIGN(4)
1786                         "1:                             \n\t"
1787                         PREFETCH" 64(%0, %%"REG_a", 4)  \n\t"
1788                         "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1789                         "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1790                         "movq %%mm0, %%mm2              \n\t" // YUYV YUYV(0)
1791                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(4)
1792                         "psrlw $8, %%mm0                \n\t" // U0V0 U0V0(0)
1793                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(4)
1794                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(0)
1795                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(4)
1796                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
1797                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
1798
1799                         MOVNTQ" %%mm2, (%1, %%"REG_a", 2)\n\t"
1800
1801                         "movq 16(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(8)
1802                         "movq 24(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(12)
1803                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(8)
1804                         "movq %%mm2, %%mm4              \n\t" // YUYV YUYV(12)
1805                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(8)
1806                         "psrlw $8, %%mm2                \n\t" // U0V0 U0V0(12)
1807                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(8)
1808                         "pand %%mm7, %%mm4              \n\t" // Y0Y0 Y0Y0(12)
1809                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
1810                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
1811
1812                         MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2)\n\t"
1813
1814                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
1815                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
1816                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
1817                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
1818                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
1819                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
1820                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
1821                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
1822
1823                         MOVNTQ" %%mm0, (%3, %%"REG_a")  \n\t"
1824                         MOVNTQ" %%mm2, (%2, %%"REG_a")  \n\t"
1825
1826                         "add $8, %%"REG_a"              \n\t"
1827                         "cmp %4, %%"REG_a"              \n\t"
1828                         " jb 1b                         \n\t"
1829                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1830                         : "memory", "%"REG_a
1831                 );
1832
1833                 ydst += lumStride;
1834                 src  += srcStride;
1835
1836                 asm volatile(
1837                         "xor %%"REG_a", %%"REG_a"       \n\t"
1838                         ASMALIGN(4)
1839                         "1:                             \n\t"
1840                         PREFETCH" 64(%0, %%"REG_a", 4)  \n\t"
1841                         "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1842                         "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1843                         "movq 16(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(8)
1844                         "movq 24(%0, %%"REG_a", 4), %%mm3\n\t" // YUYV YUYV(12)
1845                         "pand %%mm7, %%mm0              \n\t" // Y0Y0 Y0Y0(0)
1846                         "pand %%mm7, %%mm1              \n\t" // Y0Y0 Y0Y0(4)
1847                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(8)
1848                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(12)
1849                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
1850                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
1851
1852                         MOVNTQ" %%mm0, (%1, %%"REG_a", 2)\n\t"
1853                         MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2)\n\t"
1854
1855                         "add $8, %%"REG_a"              \n\t"
1856                         "cmp %4, %%"REG_a"              \n\t"
1857                         " jb 1b                         \n\t"
1858
1859                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1860                         : "memory", "%"REG_a
1861                 );
1862 #else
1863                 long i;
1864                 for(i=0; i<chromWidth; i++)
1865                 {
1866                         ydst[2*i+0]     = src[4*i+0];
1867                         udst[i]         = src[4*i+1];
1868                         ydst[2*i+1]     = src[4*i+2];
1869                         vdst[i]         = src[4*i+3];
1870                 }
1871                 ydst += lumStride;
1872                 src  += srcStride;
1873
1874                 for(i=0; i<chromWidth; i++)
1875                 {
1876                         ydst[2*i+0]     = src[4*i+0];
1877                         ydst[2*i+1]     = src[4*i+2];
1878                 }
1879 #endif
1880                 udst += chromStride;
1881                 vdst += chromStride;
1882                 ydst += lumStride;
1883                 src  += srcStride;
1884         }
1885 #ifdef HAVE_MMX
1886 asm volatile(   EMMS" \n\t"
1887                 SFENCE" \n\t"
1888                 :::"memory");
1889 #endif
1890 }
1891
1892 static inline void RENAME(yvu9toyv12)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,
1893         uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1894         long width, long height, long lumStride, long chromStride)
1895 {
1896         /* Y Plane */
1897         memcpy(ydst, ysrc, width*height);
1898
1899         /* XXX: implement upscaling for U,V */
1900 }
1901
1902 static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, long srcWidth, long srcHeight, long srcStride, long dstStride)
1903 {
1904         long x,y;
1905         
1906         dst[0]= src[0];
1907         
1908         // first line
1909         for(x=0; x<srcWidth-1; x++){
1910                 dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
1911                 dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
1912         }
1913         dst[2*srcWidth-1]= src[srcWidth-1];
1914         
1915         dst+= dstStride;
1916
1917         for(y=1; y<srcHeight; y++){
1918 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
1919                 const long mmxSize= srcWidth&~15;
1920                 asm volatile(
1921                         "mov %4, %%"REG_a"              \n\t"
1922                         "1:                             \n\t"
1923                         "movq (%0, %%"REG_a"), %%mm0    \n\t"
1924                         "movq (%1, %%"REG_a"), %%mm1    \n\t"
1925                         "movq 1(%0, %%"REG_a"), %%mm2   \n\t"
1926                         "movq 1(%1, %%"REG_a"), %%mm3   \n\t"
1927                         "movq -1(%0, %%"REG_a"), %%mm4  \n\t"
1928                         "movq -1(%1, %%"REG_a"), %%mm5  \n\t"
1929                         PAVGB" %%mm0, %%mm5             \n\t"
1930                         PAVGB" %%mm0, %%mm3             \n\t"
1931                         PAVGB" %%mm0, %%mm5             \n\t"
1932                         PAVGB" %%mm0, %%mm3             \n\t"
1933                         PAVGB" %%mm1, %%mm4             \n\t"
1934                         PAVGB" %%mm1, %%mm2             \n\t"
1935                         PAVGB" %%mm1, %%mm4             \n\t"
1936                         PAVGB" %%mm1, %%mm2             \n\t"
1937                         "movq %%mm5, %%mm7              \n\t"
1938                         "movq %%mm4, %%mm6              \n\t"
1939                         "punpcklbw %%mm3, %%mm5         \n\t"
1940                         "punpckhbw %%mm3, %%mm7         \n\t"
1941                         "punpcklbw %%mm2, %%mm4         \n\t"
1942                         "punpckhbw %%mm2, %%mm6         \n\t"
1943 #if 1
1944                         MOVNTQ" %%mm5, (%2, %%"REG_a", 2)\n\t"
1945                         MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1946                         MOVNTQ" %%mm4, (%3, %%"REG_a", 2)\n\t"
1947                         MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1948 #else
1949                         "movq %%mm5, (%2, %%"REG_a", 2) \n\t"
1950                         "movq %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1951                         "movq %%mm4, (%3, %%"REG_a", 2) \n\t"
1952                         "movq %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1953 #endif
1954                         "add $8, %%"REG_a"              \n\t"
1955                         " js 1b                         \n\t"
1956                         :: "r" (src + mmxSize  ), "r" (src + srcStride + mmxSize  ),
1957                            "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
1958                            "g" (-mmxSize)
1959                         : "%"REG_a
1960
1961                 );
1962 #else
1963                 const long mmxSize=1;
1964 #endif
1965                 dst[0        ]= (3*src[0] +   src[srcStride])>>2;
1966                 dst[dstStride]= (  src[0] + 3*src[srcStride])>>2;
1967
1968                 for(x=mmxSize-1; x<srcWidth-1; x++){
1969                         dst[2*x          +1]= (3*src[x+0] +   src[x+srcStride+1])>>2;
1970                         dst[2*x+dstStride+2]= (  src[x+0] + 3*src[x+srcStride+1])>>2;
1971                         dst[2*x+dstStride+1]= (  src[x+1] + 3*src[x+srcStride  ])>>2;
1972                         dst[2*x          +2]= (3*src[x+1] +   src[x+srcStride  ])>>2;
1973                 }
1974                 dst[srcWidth*2 -1            ]= (3*src[srcWidth-1] +   src[srcWidth-1 + srcStride])>>2;
1975                 dst[srcWidth*2 -1 + dstStride]= (  src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
1976
1977                 dst+=dstStride*2;
1978                 src+=srcStride;
1979         }
1980         
1981         // last line
1982 #if 1
1983         dst[0]= src[0];
1984         
1985         for(x=0; x<srcWidth-1; x++){
1986                 dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
1987                 dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
1988         }
1989         dst[2*srcWidth-1]= src[srcWidth-1];
1990 #else
1991         for(x=0; x<srcWidth; x++){
1992                 dst[2*x+0]=
1993                 dst[2*x+1]= src[x];
1994         }
1995 #endif
1996
1997 #ifdef HAVE_MMX
1998 asm volatile(   EMMS" \n\t"
1999                 SFENCE" \n\t"
2000                 :::"memory");
2001 #endif
2002 }
2003
2004 /**
2005  *
2006  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
2007  * problem for anyone then tell me, and ill fix it)
2008  * chrominance data is only taken from every secound line others are ignored FIXME write HQ version
2009  */
2010 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2011         long width, long height,
2012         long lumStride, long chromStride, long srcStride)
2013 {
2014         long y;
2015         const long chromWidth= width>>1;
2016         for(y=0; y<height; y+=2)
2017         {
2018 #ifdef HAVE_MMX
2019                 asm volatile(
2020                         "xorl %%eax, %%eax              \n\t"
2021                         "pcmpeqw %%mm7, %%mm7           \n\t"
2022                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
2023                         ASMALIGN(4)
2024                         "1:                             \n\t"
2025                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
2026                         "movq (%0, %%eax, 4), %%mm0     \n\t" // UYVY UYVY(0)
2027                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // UYVY UYVY(4)
2028                         "movq %%mm0, %%mm2              \n\t" // UYVY UYVY(0)
2029                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(4)
2030                         "pand %%mm7, %%mm0              \n\t" // U0V0 U0V0(0)
2031                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(4)
2032                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(0)
2033                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(4)
2034                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
2035                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
2036
2037                         MOVNTQ" %%mm2, (%1, %%eax, 2)   \n\t"
2038
2039                         "movq 16(%0, %%eax, 4), %%mm1   \n\t" // UYVY UYVY(8)
2040                         "movq 24(%0, %%eax, 4), %%mm2   \n\t" // UYVY UYVY(12)
2041                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(8)
2042                         "movq %%mm2, %%mm4              \n\t" // UYVY UYVY(12)
2043                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(8)
2044                         "pand %%mm7, %%mm2              \n\t" // U0V0 U0V0(12)
2045                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(8)
2046                         "psrlw $8, %%mm4                \n\t" // Y0Y0 Y0Y0(12)
2047                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
2048                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
2049
2050                         MOVNTQ" %%mm3, 8(%1, %%eax, 2)  \n\t"
2051
2052                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
2053                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
2054                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
2055                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
2056                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
2057                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
2058                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
2059                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
2060
2061                         MOVNTQ" %%mm0, (%3, %%eax)      \n\t"
2062                         MOVNTQ" %%mm2, (%2, %%eax)      \n\t"
2063
2064                         "addl $8, %%eax                 \n\t"
2065                         "cmpl %4, %%eax                 \n\t"
2066                         " jb 1b                         \n\t"
2067                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2068                         : "memory", "%eax"
2069                 );
2070
2071                 ydst += lumStride;
2072                 src  += srcStride;
2073
2074                 asm volatile(
2075                         "xorl %%eax, %%eax              \n\t"
2076                         ASMALIGN(4)
2077                         "1:                             \n\t"
2078                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
2079                         "movq (%0, %%eax, 4), %%mm0     \n\t" // YUYV YUYV(0)
2080                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // YUYV YUYV(4)
2081                         "movq 16(%0, %%eax, 4), %%mm2   \n\t" // YUYV YUYV(8)
2082                         "movq 24(%0, %%eax, 4), %%mm3   \n\t" // YUYV YUYV(12)
2083                         "psrlw $8, %%mm0                \n\t" // Y0Y0 Y0Y0(0)
2084                         "psrlw $8, %%mm1                \n\t" // Y0Y0 Y0Y0(4)
2085                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(8)
2086                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(12)
2087                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
2088                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
2089
2090                         MOVNTQ" %%mm0, (%1, %%eax, 2)   \n\t"
2091                         MOVNTQ" %%mm2, 8(%1, %%eax, 2)  \n\t"
2092
2093                         "addl $8, %%eax                 \n\t"
2094                         "cmpl %4, %%eax                 \n\t"
2095                         " jb 1b                         \n\t"
2096
2097                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2098                         : "memory", "%eax"
2099                 );
2100 #else
2101                 long i;
2102                 for(i=0; i<chromWidth; i++)
2103                 {
2104                         udst[i]         = src[4*i+0];
2105                         ydst[2*i+0]     = src[4*i+1];
2106                         vdst[i]         = src[4*i+2];
2107                         ydst[2*i+1]     = src[4*i+3];
2108                 }
2109                 ydst += lumStride;
2110                 src  += srcStride;
2111
2112                 for(i=0; i<chromWidth; i++)
2113                 {
2114                         ydst[2*i+0]     = src[4*i+1];
2115                         ydst[2*i+1]     = src[4*i+3];
2116                 }
2117 #endif
2118                 udst += chromStride;
2119                 vdst += chromStride;
2120                 ydst += lumStride;
2121                 src  += srcStride;
2122         }
2123 #ifdef HAVE_MMX
2124 asm volatile(   EMMS" \n\t"
2125                 SFENCE" \n\t"
2126                 :::"memory");
2127 #endif
2128 }
2129
2130 /**
2131  *
2132  * height should be a multiple of 2 and width should be a multiple of 2 (if this is a
2133  * problem for anyone then tell me, and ill fix it)
2134  * chrominance data is only taken from every secound line others are ignored in the C version FIXME write HQ version
2135  */
2136 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2137         long width, long height,
2138         long lumStride, long chromStride, long srcStride)
2139 {
2140         long y;
2141         const long chromWidth= width>>1;
2142 #ifdef HAVE_MMX
2143         for(y=0; y<height-2; y+=2)
2144         {
2145                 long i;
2146                 for(i=0; i<2; i++)
2147                 {
2148                         asm volatile(
2149                                 "mov %2, %%"REG_a"              \n\t"
2150                                 "movq "MANGLE(bgr2YCoeff)", %%mm6               \n\t"
2151                                 "movq "MANGLE(w1111)", %%mm5            \n\t"
2152                                 "pxor %%mm7, %%mm7              \n\t"
2153                                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2154                                 ASMALIGN(4)
2155                                 "1:                             \n\t"
2156                                 PREFETCH" 64(%0, %%"REG_d")     \n\t"
2157                                 "movd (%0, %%"REG_d"), %%mm0    \n\t"
2158                                 "movd 3(%0, %%"REG_d"), %%mm1   \n\t"
2159                                 "punpcklbw %%mm7, %%mm0         \n\t"
2160                                 "punpcklbw %%mm7, %%mm1         \n\t"
2161                                 "movd 6(%0, %%"REG_d"), %%mm2   \n\t"
2162                                 "movd 9(%0, %%"REG_d"), %%mm3   \n\t"
2163                                 "punpcklbw %%mm7, %%mm2         \n\t"
2164                                 "punpcklbw %%mm7, %%mm3         \n\t"
2165                                 "pmaddwd %%mm6, %%mm0           \n\t"
2166                                 "pmaddwd %%mm6, %%mm1           \n\t"
2167                                 "pmaddwd %%mm6, %%mm2           \n\t"
2168                                 "pmaddwd %%mm6, %%mm3           \n\t"
2169 #ifndef FAST_BGR2YV12
2170                                 "psrad $8, %%mm0                \n\t"
2171                                 "psrad $8, %%mm1                \n\t"
2172                                 "psrad $8, %%mm2                \n\t"
2173                                 "psrad $8, %%mm3                \n\t"
2174 #endif
2175                                 "packssdw %%mm1, %%mm0          \n\t"
2176                                 "packssdw %%mm3, %%mm2          \n\t"
2177                                 "pmaddwd %%mm5, %%mm0           \n\t"
2178                                 "pmaddwd %%mm5, %%mm2           \n\t"
2179                                 "packssdw %%mm2, %%mm0          \n\t"
2180                                 "psraw $7, %%mm0                \n\t"
2181
2182                                 "movd 12(%0, %%"REG_d"), %%mm4  \n\t"
2183                                 "movd 15(%0, %%"REG_d"), %%mm1  \n\t"
2184                                 "punpcklbw %%mm7, %%mm4         \n\t"
2185                                 "punpcklbw %%mm7, %%mm1         \n\t"
2186                                 "movd 18(%0, %%"REG_d"), %%mm2  \n\t"
2187                                 "movd 21(%0, %%"REG_d"), %%mm3  \n\t"
2188                                 "punpcklbw %%mm7, %%mm2         \n\t"
2189                                 "punpcklbw %%mm7, %%mm3         \n\t"
2190                                 "pmaddwd %%mm6, %%mm4           \n\t"
2191                                 "pmaddwd %%mm6, %%mm1           \n\t"
2192                                 "pmaddwd %%mm6, %%mm2           \n\t"
2193                                 "pmaddwd %%mm6, %%mm3           \n\t"
2194 #ifndef FAST_BGR2YV12
2195                                 "psrad $8, %%mm4                \n\t"
2196                                 "psrad $8, %%mm1                \n\t"
2197                                 "psrad $8, %%mm2                \n\t"
2198                                 "psrad $8, %%mm3                \n\t"
2199 #endif
2200                                 "packssdw %%mm1, %%mm4          \n\t"
2201                                 "packssdw %%mm3, %%mm2          \n\t"
2202                                 "pmaddwd %%mm5, %%mm4           \n\t"
2203                                 "pmaddwd %%mm5, %%mm2           \n\t"
2204                                 "add $24, %%"REG_d"             \n\t"
2205                                 "packssdw %%mm2, %%mm4          \n\t"
2206                                 "psraw $7, %%mm4                \n\t"
2207
2208                                 "packuswb %%mm4, %%mm0          \n\t"
2209                                 "paddusb "MANGLE(bgr2YOffset)", %%mm0   \n\t"
2210
2211                                 MOVNTQ" %%mm0, (%1, %%"REG_a")  \n\t"
2212                                 "add $8, %%"REG_a"              \n\t"
2213                                 " js 1b                         \n\t"
2214                                 : : "r" (src+width*3), "r" (ydst+width), "g" (-width)
2215                                 : "%"REG_a, "%"REG_d
2216                         );
2217                         ydst += lumStride;
2218                         src  += srcStride;
2219                 }
2220                 src -= srcStride*2;
2221                 asm volatile(
2222                         "mov %4, %%"REG_a"              \n\t"
2223                         "movq "MANGLE(w1111)", %%mm5            \n\t"
2224                         "movq "MANGLE(bgr2UCoeff)", %%mm6               \n\t"
2225                         "pxor %%mm7, %%mm7              \n\t"
2226                         "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2227                         "add %%"REG_d", %%"REG_d"       \n\t"
2228                         ASMALIGN(4)
2229                         "1:                             \n\t"
2230                         PREFETCH" 64(%0, %%"REG_d")     \n\t"
2231                         PREFETCH" 64(%1, %%"REG_d")     \n\t"
2232 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2233                         "movq (%0, %%"REG_d"), %%mm0    \n\t"
2234                         "movq (%1, %%"REG_d"), %%mm1    \n\t"
2235                         "movq 6(%0, %%"REG_d"), %%mm2   \n\t"
2236                         "movq 6(%1, %%"REG_d"), %%mm3   \n\t"
2237                         PAVGB" %%mm1, %%mm0             \n\t"
2238                         PAVGB" %%mm3, %%mm2             \n\t"
2239                         "movq %%mm0, %%mm1              \n\t"
2240                         "movq %%mm2, %%mm3              \n\t"
2241                         "psrlq $24, %%mm0               \n\t"
2242                         "psrlq $24, %%mm2               \n\t"
2243                         PAVGB" %%mm1, %%mm0             \n\t"
2244                         PAVGB" %%mm3, %%mm2             \n\t"
2245                         "punpcklbw %%mm7, %%mm0         \n\t"
2246                         "punpcklbw %%mm7, %%mm2         \n\t"
2247 #else
2248                         "movd (%0, %%"REG_d"), %%mm0    \n\t"
2249                         "movd (%1, %%"REG_d"), %%mm1    \n\t"
2250                         "movd 3(%0, %%"REG_d"), %%mm2   \n\t"
2251                         "movd 3(%1, %%"REG_d"), %%mm3   \n\t"
2252                         "punpcklbw %%mm7, %%mm0         \n\t"
2253                         "punpcklbw %%mm7, %%mm1         \n\t"
2254                         "punpcklbw %%mm7, %%mm2         \n\t"
2255                         "punpcklbw %%mm7, %%mm3         \n\t"
2256                         "paddw %%mm1, %%mm0             \n\t"
2257                         "paddw %%mm3, %%mm2             \n\t"
2258                         "paddw %%mm2, %%mm0             \n\t"
2259                         "movd 6(%0, %%"REG_d"), %%mm4   \n\t"
2260                         "movd 6(%1, %%"REG_d"), %%mm1   \n\t"
2261                         "movd 9(%0, %%"REG_d"), %%mm2   \n\t"
2262                         "movd 9(%1, %%"REG_d"), %%mm3   \n\t"
2263                         "punpcklbw %%mm7, %%mm4         \n\t"
2264                         "punpcklbw %%mm7, %%mm1         \n\t"
2265                         "punpcklbw %%mm7, %%mm2         \n\t"
2266                         "punpcklbw %%mm7, %%mm3         \n\t"
2267                         "paddw %%mm1, %%mm4             \n\t"
2268                         "paddw %%mm3, %%mm2             \n\t"
2269                         "paddw %%mm4, %%mm2             \n\t"
2270                         "psrlw $2, %%mm0                \n\t"
2271                         "psrlw $2, %%mm2                \n\t"
2272 #endif
2273                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
2274                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
2275
2276                         "pmaddwd %%mm0, %%mm1           \n\t"
2277                         "pmaddwd %%mm2, %%mm3           \n\t"
2278                         "pmaddwd %%mm6, %%mm0           \n\t"
2279                         "pmaddwd %%mm6, %%mm2           \n\t"
2280 #ifndef FAST_BGR2YV12
2281                         "psrad $8, %%mm0                \n\t"
2282                         "psrad $8, %%mm1                \n\t"
2283                         "psrad $8, %%mm2                \n\t"
2284                         "psrad $8, %%mm3                \n\t"
2285 #endif
2286                         "packssdw %%mm2, %%mm0          \n\t"
2287                         "packssdw %%mm3, %%mm1          \n\t"
2288                         "pmaddwd %%mm5, %%mm0           \n\t"
2289                         "pmaddwd %%mm5, %%mm1           \n\t"
2290                         "packssdw %%mm1, %%mm0          \n\t" // V1 V0 U1 U0
2291                         "psraw $7, %%mm0                \n\t"
2292
2293 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2294                         "movq 12(%0, %%"REG_d"), %%mm4  \n\t"
2295                         "movq 12(%1, %%"REG_d"), %%mm1  \n\t"
2296                         "movq 18(%0, %%"REG_d"), %%mm2  \n\t"
2297                         "movq 18(%1, %%"REG_d"), %%mm3  \n\t"
2298                         PAVGB" %%mm1, %%mm4             \n\t"
2299                         PAVGB" %%mm3, %%mm2             \n\t"
2300                         "movq %%mm4, %%mm1              \n\t"
2301                         "movq %%mm2, %%mm3              \n\t"
2302                         "psrlq $24, %%mm4               \n\t"
2303                         "psrlq $24, %%mm2               \n\t"
2304                         PAVGB" %%mm1, %%mm4             \n\t"
2305                         PAVGB" %%mm3, %%mm2             \n\t"
2306                         "punpcklbw %%mm7, %%mm4         \n\t"
2307                         "punpcklbw %%mm7, %%mm2         \n\t"
2308 #else
2309                         "movd 12(%0, %%"REG_d"), %%mm4  \n\t"
2310                         "movd 12(%1, %%"REG_d"), %%mm1  \n\t"
2311                         "movd 15(%0, %%"REG_d"), %%mm2  \n\t"
2312                         "movd 15(%1, %%"REG_d"), %%mm3  \n\t"
2313                         "punpcklbw %%mm7, %%mm4         \n\t"
2314                         "punpcklbw %%mm7, %%mm1         \n\t"
2315                         "punpcklbw %%mm7, %%mm2         \n\t"
2316                         "punpcklbw %%mm7, %%mm3         \n\t"
2317                         "paddw %%mm1, %%mm4             \n\t"
2318                         "paddw %%mm3, %%mm2             \n\t"
2319                         "paddw %%mm2, %%mm4             \n\t"
2320                         "movd 18(%0, %%"REG_d"), %%mm5  \n\t"
2321                         "movd 18(%1, %%"REG_d"), %%mm1  \n\t"
2322                         "movd 21(%0, %%"REG_d"), %%mm2  \n\t"
2323                         "movd 21(%1, %%"REG_d"), %%mm3  \n\t"
2324                         "punpcklbw %%mm7, %%mm5         \n\t"
2325                         "punpcklbw %%mm7, %%mm1         \n\t"
2326                         "punpcklbw %%mm7, %%mm2         \n\t"
2327                         "punpcklbw %%mm7, %%mm3         \n\t"
2328                         "paddw %%mm1, %%mm5             \n\t"
2329                         "paddw %%mm3, %%mm2             \n\t"
2330                         "paddw %%mm5, %%mm2             \n\t"
2331                         "movq "MANGLE(w1111)", %%mm5            \n\t"
2332                         "psrlw $2, %%mm4                \n\t"
2333                         "psrlw $2, %%mm2                \n\t"
2334 #endif
2335                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
2336                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
2337
2338                         "pmaddwd %%mm4, %%mm1           \n\t"
2339                         "pmaddwd %%mm2, %%mm3           \n\t"
2340                         "pmaddwd %%mm6, %%mm4           \n\t"
2341                         "pmaddwd %%mm6, %%mm2           \n\t"
2342 #ifndef FAST_BGR2YV12
2343                         "psrad $8, %%mm4                \n\t"
2344                         "psrad $8, %%mm1                \n\t"
2345                         "psrad $8, %%mm2                \n\t"
2346                         "psrad $8, %%mm3                \n\t"
2347 #endif
2348                         "packssdw %%mm2, %%mm4          \n\t"
2349                         "packssdw %%mm3, %%mm1          \n\t"
2350                         "pmaddwd %%mm5, %%mm4           \n\t"
2351                         "pmaddwd %%mm5, %%mm1           \n\t"
2352                         "add $24, %%"REG_d"             \n\t"
2353                         "packssdw %%mm1, %%mm4          \n\t" // V3 V2 U3 U2
2354                         "psraw $7, %%mm4                \n\t"
2355
2356                         "movq %%mm0, %%mm1              \n\t"
2357                         "punpckldq %%mm4, %%mm0         \n\t"
2358                         "punpckhdq %%mm4, %%mm1         \n\t"
2359                         "packsswb %%mm1, %%mm0          \n\t"
2360                         "paddb "MANGLE(bgr2UVOffset)", %%mm0    \n\t"
2361                         "movd %%mm0, (%2, %%"REG_a")    \n\t"
2362                         "punpckhdq %%mm0, %%mm0         \n\t"
2363                         "movd %%mm0, (%3, %%"REG_a")    \n\t"
2364                         "add $4, %%"REG_a"              \n\t"
2365                         " js 1b                         \n\t"
2366                         : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth)
2367                         : "%"REG_a, "%"REG_d
2368                 );
2369
2370                 udst += chromStride;
2371                 vdst += chromStride;
2372                 src  += srcStride*2;
2373         }
2374
2375         asm volatile(   EMMS" \n\t"
2376                         SFENCE" \n\t"
2377                         :::"memory");
2378 #else
2379         y=0;
2380 #endif
2381         for(; y<height; y+=2)
2382         {
2383                 long i;
2384                 for(i=0; i<chromWidth; i++)
2385                 {
2386                         unsigned int b= src[6*i+0];
2387                         unsigned int g= src[6*i+1];
2388                         unsigned int r= src[6*i+2];
2389
2390                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2391                         unsigned int V  =  ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128;
2392                         unsigned int U  =  ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128;
2393
2394                         udst[i]         = U;
2395                         vdst[i]         = V;
2396                         ydst[2*i]       = Y;
2397
2398                         b= src[6*i+3];
2399                         g= src[6*i+4];
2400                         r= src[6*i+5];
2401
2402                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2403                         ydst[2*i+1]     = Y;
2404                 }
2405                 ydst += lumStride;
2406                 src  += srcStride;
2407
2408                 for(i=0; i<chromWidth; i++)
2409                 {
2410                         unsigned int b= src[6*i+0];
2411                         unsigned int g= src[6*i+1];
2412                         unsigned int r= src[6*i+2];
2413
2414                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2415
2416                         ydst[2*i]       = Y;
2417
2418                         b= src[6*i+3];
2419                         g= src[6*i+4];
2420                         r= src[6*i+5];
2421
2422                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2423                         ydst[2*i+1]     = Y;
2424                 }
2425                 udst += chromStride;
2426                 vdst += chromStride;
2427                 ydst += lumStride;
2428                 src  += srcStride;
2429         }
2430 }
2431
2432 void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest,
2433                             long width, long height, long src1Stride,
2434                             long src2Stride, long dstStride){
2435         long h;
2436
2437         for(h=0; h < height; h++)
2438         {
2439                 long w;
2440
2441 #ifdef HAVE_MMX
2442 #ifdef HAVE_SSE2
2443                 asm(
2444                         "xor %%"REG_a", %%"REG_a"       \n\t"
2445                         "1:                             \n\t"
2446                         PREFETCH" 64(%1, %%"REG_a")     \n\t"
2447                         PREFETCH" 64(%2, %%"REG_a")     \n\t"
2448                         "movdqa (%1, %%"REG_a"), %%xmm0 \n\t"
2449                         "movdqa (%1, %%"REG_a"), %%xmm1 \n\t"
2450                         "movdqa (%2, %%"REG_a"), %%xmm2 \n\t"
2451                         "punpcklbw %%xmm2, %%xmm0       \n\t"
2452                         "punpckhbw %%xmm2, %%xmm1       \n\t"
2453                         "movntdq %%xmm0, (%0, %%"REG_a", 2)\n\t"
2454                         "movntdq %%xmm1, 16(%0, %%"REG_a", 2)\n\t"
2455                         "add $16, %%"REG_a"             \n\t"
2456                         "cmp %3, %%"REG_a"              \n\t"
2457                         " jb 1b                         \n\t"
2458                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2459                         : "memory", "%"REG_a""
2460                 );
2461 #else
2462                 asm(
2463                         "xor %%"REG_a", %%"REG_a"       \n\t"
2464                         "1:                             \n\t"
2465                         PREFETCH" 64(%1, %%"REG_a")     \n\t"
2466                         PREFETCH" 64(%2, %%"REG_a")     \n\t"
2467                         "movq (%1, %%"REG_a"), %%mm0    \n\t"
2468                         "movq 8(%1, %%"REG_a"), %%mm2   \n\t"
2469                         "movq %%mm0, %%mm1              \n\t"
2470                         "movq %%mm2, %%mm3              \n\t"
2471                         "movq (%2, %%"REG_a"), %%mm4    \n\t"
2472                         "movq 8(%2, %%"REG_a"), %%mm5   \n\t"
2473                         "punpcklbw %%mm4, %%mm0         \n\t"
2474                         "punpckhbw %%mm4, %%mm1         \n\t"
2475                         "punpcklbw %%mm5, %%mm2         \n\t"
2476                         "punpckhbw %%mm5, %%mm3         \n\t"
2477                         MOVNTQ" %%mm0, (%0, %%"REG_a", 2)\n\t"
2478                         MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2)\n\t"
2479                         MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2)\n\t"
2480                         MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2)\n\t"
2481                         "add $16, %%"REG_a"             \n\t"
2482                         "cmp %3, %%"REG_a"              \n\t"
2483                         " jb 1b                         \n\t"
2484                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2485                         : "memory", "%"REG_a
2486                 );
2487 #endif
2488                 for(w= (width&(~15)); w < width; w++)
2489                 {
2490                         dest[2*w+0] = src1[w];
2491                         dest[2*w+1] = src2[w];
2492                 }
2493 #else
2494                 for(w=0; w < width; w++)
2495                 {
2496                         dest[2*w+0] = src1[w];
2497                         dest[2*w+1] = src2[w];
2498                 }
2499 #endif
2500                 dest += dstStride;
2501                 src1 += src1Stride;
2502                 src2 += src2Stride;
2503         }
2504 #ifdef HAVE_MMX
2505         asm(
2506                 EMMS" \n\t"
2507                 SFENCE" \n\t"
2508                 ::: "memory"
2509                 );
2510 #endif
2511 }
2512
2513 static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
2514                         uint8_t *dst1, uint8_t *dst2,
2515                         long width, long height,
2516                         long srcStride1, long srcStride2,
2517                         long dstStride1, long dstStride2)
2518 {
2519     long y,x,w,h;
2520     w=width/2; h=height/2;
2521 #ifdef HAVE_MMX
2522     asm volatile(
2523         PREFETCH" %0\n\t"
2524         PREFETCH" %1\n\t"
2525         ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
2526 #endif
2527     for(y=0;y<h;y++){
2528         const uint8_t* s1=src1+srcStride1*(y>>1);
2529         uint8_t* d=dst1+dstStride1*y;
2530         x=0;
2531 #ifdef HAVE_MMX
2532         for(;x<w-31;x+=32)
2533         {
2534             asm volatile(
2535                 PREFETCH" 32%1\n\t"
2536                 "movq   %1, %%mm0\n\t"
2537                 "movq   8%1, %%mm2\n\t"
2538                 "movq   16%1, %%mm4\n\t"
2539                 "movq   24%1, %%mm6\n\t"
2540                 "movq   %%mm0, %%mm1\n\t"
2541                 "movq   %%mm2, %%mm3\n\t"
2542                 "movq   %%mm4, %%mm5\n\t"
2543                 "movq   %%mm6, %%mm7\n\t"
2544                 "punpcklbw %%mm0, %%mm0\n\t"
2545                 "punpckhbw %%mm1, %%mm1\n\t"
2546                 "punpcklbw %%mm2, %%mm2\n\t"
2547                 "punpckhbw %%mm3, %%mm3\n\t"
2548                 "punpcklbw %%mm4, %%mm4\n\t"
2549                 "punpckhbw %%mm5, %%mm5\n\t"
2550                 "punpcklbw %%mm6, %%mm6\n\t"
2551                 "punpckhbw %%mm7, %%mm7\n\t"
2552                 MOVNTQ" %%mm0, %0\n\t"
2553                 MOVNTQ" %%mm1, 8%0\n\t"
2554                 MOVNTQ" %%mm2, 16%0\n\t"
2555                 MOVNTQ" %%mm3, 24%0\n\t"
2556                 MOVNTQ" %%mm4, 32%0\n\t"
2557                 MOVNTQ" %%mm5, 40%0\n\t"
2558                 MOVNTQ" %%mm6, 48%0\n\t"
2559                 MOVNTQ" %%mm7, 56%0"
2560                 :"=m"(d[2*x])
2561                 :"m"(s1[x])
2562                 :"memory");
2563         }
2564 #endif
2565         for(;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
2566     }
2567     for(y=0;y<h;y++){
2568         const uint8_t* s2=src2+srcStride2*(y>>1);
2569         uint8_t* d=dst2+dstStride2*y;
2570         x=0;
2571 #ifdef HAVE_MMX
2572         for(;x<w-31;x+=32)
2573         {
2574             asm volatile(
2575                 PREFETCH" 32%1\n\t"
2576                 "movq   %1, %%mm0\n\t"
2577                 "movq   8%1, %%mm2\n\t"
2578                 "movq   16%1, %%mm4\n\t"
2579                 "movq   24%1, %%mm6\n\t"
2580                 "movq   %%mm0, %%mm1\n\t"
2581                 "movq   %%mm2, %%mm3\n\t"
2582                 "movq   %%mm4, %%mm5\n\t"
2583                 "movq   %%mm6, %%mm7\n\t"
2584                 "punpcklbw %%mm0, %%mm0\n\t"
2585                 "punpckhbw %%mm1, %%mm1\n\t"
2586                 "punpcklbw %%mm2, %%mm2\n\t"
2587                 "punpckhbw %%mm3, %%mm3\n\t"
2588                 "punpcklbw %%mm4, %%mm4\n\t"
2589                 "punpckhbw %%mm5, %%mm5\n\t"
2590                 "punpcklbw %%mm6, %%mm6\n\t"
2591                 "punpckhbw %%mm7, %%mm7\n\t"
2592                 MOVNTQ" %%mm0, %0\n\t"
2593                 MOVNTQ" %%mm1, 8%0\n\t"
2594                 MOVNTQ" %%mm2, 16%0\n\t"
2595                 MOVNTQ" %%mm3, 24%0\n\t"
2596                 MOVNTQ" %%mm4, 32%0\n\t"
2597                 MOVNTQ" %%mm5, 40%0\n\t"
2598                 MOVNTQ" %%mm6, 48%0\n\t"
2599                 MOVNTQ" %%mm7, 56%0"
2600                 :"=m"(d[2*x])
2601                 :"m"(s2[x])
2602                 :"memory");
2603         }
2604 #endif
2605         for(;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
2606     }
2607 #ifdef HAVE_MMX
2608         asm(
2609                 EMMS" \n\t"
2610                 SFENCE" \n\t"
2611                 ::: "memory"
2612                 );
2613 #endif
2614 }
2615
2616 static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
2617                         uint8_t *dst,
2618                         long width, long height,
2619                         long srcStride1, long srcStride2,
2620                         long srcStride3, long dstStride)
2621 {
2622     long y,x,w,h;
2623     w=width/2; h=height;
2624     for(y=0;y<h;y++){
2625         const uint8_t* yp=src1+srcStride1*y;
2626         const uint8_t* up=src2+srcStride2*(y>>2);
2627         const uint8_t* vp=src3+srcStride3*(y>>2);
2628         uint8_t* d=dst+dstStride*y;
2629         x=0;
2630 #ifdef HAVE_MMX
2631         for(;x<w-7;x+=8)
2632         {
2633             asm volatile(
2634                 PREFETCH" 32(%1, %0)\n\t"
2635                 PREFETCH" 32(%2, %0)\n\t"
2636                 PREFETCH" 32(%3, %0)\n\t"
2637                 "movq   (%1, %0, 4), %%mm0\n\t"       /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2638                 "movq   (%2, %0), %%mm1\n\t"       /* U0U1U2U3U4U5U6U7 */
2639                 "movq   (%3, %0), %%mm2\n\t"         /* V0V1V2V3V4V5V6V7 */
2640                 "movq   %%mm0, %%mm3\n\t"    /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2641                 "movq   %%mm1, %%mm4\n\t"    /* U0U1U2U3U4U5U6U7 */
2642                 "movq   %%mm2, %%mm5\n\t"    /* V0V1V2V3V4V5V6V7 */
2643                 "punpcklbw %%mm1, %%mm1\n\t" /* U0U0 U1U1 U2U2 U3U3 */
2644                 "punpcklbw %%mm2, %%mm2\n\t" /* V0V0 V1V1 V2V2 V3V3 */
2645                 "punpckhbw %%mm4, %%mm4\n\t" /* U4U4 U5U5 U6U6 U7U7 */
2646                 "punpckhbw %%mm5, %%mm5\n\t" /* V4V4 V5V5 V6V6 V7V7 */
2647
2648                 "movq   %%mm1, %%mm6\n\t"
2649                 "punpcklbw %%mm2, %%mm1\n\t" /* U0V0 U0V0 U1V1 U1V1*/
2650                 "punpcklbw %%mm1, %%mm0\n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
2651                 "punpckhbw %%mm1, %%mm3\n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
2652                 MOVNTQ" %%mm0, (%4, %0, 8)\n\t"
2653                 MOVNTQ" %%mm3, 8(%4, %0, 8)\n\t"
2654                 
2655                 "punpckhbw %%mm2, %%mm6\n\t" /* U2V2 U2V2 U3V3 U3V3*/
2656                 "movq   8(%1, %0, 4), %%mm0\n\t"
2657                 "movq   %%mm0, %%mm3\n\t"
2658                 "punpcklbw %%mm6, %%mm0\n\t" /* Y U2 Y V2 Y U2 Y V2*/
2659                 "punpckhbw %%mm6, %%mm3\n\t" /* Y U3 Y V3 Y U3 Y V3*/
2660                 MOVNTQ" %%mm0, 16(%4, %0, 8)\n\t"
2661                 MOVNTQ" %%mm3, 24(%4, %0, 8)\n\t"
2662
2663                 "movq   %%mm4, %%mm6\n\t"
2664                 "movq   16(%1, %0, 4), %%mm0\n\t"
2665                 "movq   %%mm0, %%mm3\n\t"
2666                 "punpcklbw %%mm5, %%mm4\n\t"
2667                 "punpcklbw %%mm4, %%mm0\n\t" /* Y U4 Y V4 Y U4 Y V4*/
2668                 "punpckhbw %%mm4, %%mm3\n\t" /* Y U5 Y V5 Y U5 Y V5*/
2669                 MOVNTQ" %%mm0, 32(%4, %0, 8)\n\t"
2670                 MOVNTQ" %%mm3, 40(%4, %0, 8)\n\t"
2671                 
2672                 "punpckhbw %%mm5, %%mm6\n\t"
2673                 "movq   24(%1, %0, 4), %%mm0\n\t"
2674                 "movq   %%mm0, %%mm3\n\t"
2675                 "punpcklbw %%mm6, %%mm0\n\t" /* Y U6 Y V6 Y U6 Y V6*/
2676                 "punpckhbw %%mm6, %%mm3\n\t" /* Y U7 Y V7 Y U7 Y V7*/
2677                 MOVNTQ" %%mm0, 48(%4, %0, 8)\n\t"
2678                 MOVNTQ" %%mm3, 56(%4, %0, 8)\n\t"
2679
2680                 : "+r" (x)
2681                 : "r"(yp), "r" (up), "r"(vp), "r"(d)
2682                 :"memory");
2683         }
2684 #endif
2685         for(; x<w; x++)
2686         {
2687             const long x2= x<<2;
2688             d[8*x+0]=yp[x2];
2689             d[8*x+1]=up[x];
2690             d[8*x+2]=yp[x2+1];
2691             d[8*x+3]=vp[x];
2692             d[8*x+4]=yp[x2+2];
2693             d[8*x+5]=up[x];
2694             d[8*x+6]=yp[x2+3];
2695             d[8*x+7]=vp[x];
2696         }
2697     }
2698 #ifdef HAVE_MMX
2699         asm(
2700                 EMMS" \n\t"
2701                 SFENCE" \n\t"
2702                 ::: "memory"
2703                 );
2704 #endif
2705 }
2706
2707 static inline void RENAME(rgb2rgb_init)(void){
2708         rgb15to16= RENAME(rgb15to16);
2709         rgb15to24= RENAME(rgb15to24);
2710         rgb15to32= RENAME(rgb15to32);
2711         rgb16to24= RENAME(rgb16to24);
2712         rgb16to32= RENAME(rgb16to32);
2713         rgb16to15= RENAME(rgb16to15);
2714         rgb24to16= RENAME(rgb24to16);
2715         rgb24to15= RENAME(rgb24to15);
2716         rgb24to32= RENAME(rgb24to32);
2717         rgb32to16= RENAME(rgb32to16);
2718         rgb32to15= RENAME(rgb32to15);
2719         rgb32to24= RENAME(rgb32to24);
2720         rgb24tobgr15= RENAME(rgb24tobgr15);
2721         rgb24tobgr16= RENAME(rgb24tobgr16);
2722         rgb24tobgr24= RENAME(rgb24tobgr24);
2723         rgb32tobgr32= RENAME(rgb32tobgr32);
2724         rgb32tobgr16= RENAME(rgb32tobgr16);
2725         rgb32tobgr15= RENAME(rgb32tobgr15);
2726         yv12toyuy2= RENAME(yv12toyuy2);
2727         yv12touyvy= RENAME(yv12touyvy);
2728         yuv422ptoyuy2= RENAME(yuv422ptoyuy2);
2729         yuy2toyv12= RENAME(yuy2toyv12);
2730 //      uyvytoyv12= RENAME(uyvytoyv12);
2731 //      yvu9toyv12= RENAME(yvu9toyv12);
2732         planar2x= RENAME(planar2x);
2733         rgb24toyv12= RENAME(rgb24toyv12);
2734         interleaveBytes= RENAME(interleaveBytes);
2735         vu9_to_vu12= RENAME(vu9_to_vu12);
2736         yvu9_to_yuy2= RENAME(yvu9_to_yuy2);
2737 }