skip MMX code in rgb32to16 if the size of the input is smaller than the
[ffmpeg.git] / libswscale / rgb2rgb_template.c
1 /*
2  *
3  *  rgb2rgb.c, Software RGB to RGB convertor
4  *  pluralize by Software PAL8 to RGB convertor
5  *               Software YUV to YUV convertor
6  *               Software YUV to RGB convertor
7  *  Written by Nick Kurshev.
8  *  palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
9  *  lot of big-endian byteorder fixes by Alex Beregszaszi
10  *
11  * This file is part of FFmpeg.
12  *
13  * FFmpeg is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  * FFmpeg is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21  * GNU General Public License for more details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with FFmpeg; if not, write to the Free Software
25  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26  * 
27  * the C code (not assembly, mmx, ...) of this file can be used
28  * under the LGPL license too
29  */
30
31 #include <stddef.h>
32 #include <inttypes.h> /* for __WORDSIZE */
33
34 #ifndef __WORDSIZE
35 // #warning You have misconfigured system and probably will lose performance!
36 #define __WORDSIZE MP_WORDSIZE
37 #endif
38
39 #undef PREFETCH
40 #undef MOVNTQ
41 #undef EMMS
42 #undef SFENCE
43 #undef MMREG_SIZE
44 #undef PREFETCHW
45 #undef PAVGB
46
47 #ifdef HAVE_SSE2
48 #define MMREG_SIZE 16
49 #else
50 #define MMREG_SIZE 8
51 #endif
52
53 #ifdef HAVE_3DNOW
54 #define PREFETCH  "prefetch"
55 #define PREFETCHW "prefetchw"
56 #define PAVGB     "pavgusb"
57 #elif defined ( HAVE_MMX2 )
58 #define PREFETCH "prefetchnta"
59 #define PREFETCHW "prefetcht0"
60 #define PAVGB     "pavgb"
61 #else
62 #ifdef __APPLE__
63 #define PREFETCH "#"
64 #define PREFETCHW "#"
65 #else
66 #define PREFETCH  " # nop"
67 #define PREFETCHW " # nop"
68 #endif
69 #endif
70
71 #ifdef HAVE_3DNOW
72 /* On K6 femms is faster of emms. On K7 femms is directly mapped on emms. */
73 #define EMMS     "femms"
74 #else
75 #define EMMS     "emms"
76 #endif
77
78 #ifdef HAVE_MMX2
79 #define MOVNTQ "movntq"
80 #define SFENCE "sfence"
81 #else
82 #define MOVNTQ "movq"
83 #define SFENCE " # nop"
84 #endif
85
86 static inline void RENAME(rgb24to32)(const uint8_t *src,uint8_t *dst,long src_size)
87 {
88   uint8_t *dest = dst;
89   const uint8_t *s = src;
90   const uint8_t *end;
91 #ifdef HAVE_MMX
92   const uint8_t *mm_end;
93 #endif
94   end = s + src_size;
95 #ifdef HAVE_MMX
96   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
97   mm_end = end - 23;
98   __asm __volatile("movq        %0, %%mm7"::"m"(mask32):"memory");
99   while(s < mm_end)
100   {
101     __asm __volatile(
102         PREFETCH"       32%1\n\t"
103         "movd   %1, %%mm0\n\t"
104         "punpckldq 3%1, %%mm0\n\t"
105         "movd   6%1, %%mm1\n\t"
106         "punpckldq 9%1, %%mm1\n\t"
107         "movd   12%1, %%mm2\n\t"
108         "punpckldq 15%1, %%mm2\n\t"
109         "movd   18%1, %%mm3\n\t"
110         "punpckldq 21%1, %%mm3\n\t"
111         "pand   %%mm7, %%mm0\n\t"
112         "pand   %%mm7, %%mm1\n\t"
113         "pand   %%mm7, %%mm2\n\t"
114         "pand   %%mm7, %%mm3\n\t"
115         MOVNTQ" %%mm0, %0\n\t"
116         MOVNTQ" %%mm1, 8%0\n\t"
117         MOVNTQ" %%mm2, 16%0\n\t"
118         MOVNTQ" %%mm3, 24%0"
119         :"=m"(*dest)
120         :"m"(*s)
121         :"memory");
122     dest += 32;
123     s += 24;
124   }
125   __asm __volatile(SFENCE:::"memory");
126   __asm __volatile(EMMS:::"memory");
127 #endif
128   while(s < end)
129   {
130 #ifdef WORDS_BIGENDIAN
131     /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */
132     *dest++ = 0;
133     *dest++ = s[2];
134     *dest++ = s[1];
135     *dest++ = s[0];
136     s+=3;
137 #else
138     *dest++ = *s++;
139     *dest++ = *s++;
140     *dest++ = *s++;
141     *dest++ = 0;
142 #endif
143   }
144 }
145
146 static inline void RENAME(rgb32to24)(const uint8_t *src,uint8_t *dst,long src_size)
147 {
148   uint8_t *dest = dst;
149   const uint8_t *s = src;
150   const uint8_t *end;
151 #ifdef HAVE_MMX
152   const uint8_t *mm_end;
153 #endif
154   end = s + src_size;
155 #ifdef HAVE_MMX
156   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
157   mm_end = end - 31;
158   while(s < mm_end)
159   {
160     __asm __volatile(
161         PREFETCH"       32%1\n\t"
162         "movq   %1, %%mm0\n\t"
163         "movq   8%1, %%mm1\n\t"
164         "movq   16%1, %%mm4\n\t"
165         "movq   24%1, %%mm5\n\t"
166         "movq   %%mm0, %%mm2\n\t"
167         "movq   %%mm1, %%mm3\n\t"
168         "movq   %%mm4, %%mm6\n\t"
169         "movq   %%mm5, %%mm7\n\t"
170         "psrlq  $8, %%mm2\n\t"
171         "psrlq  $8, %%mm3\n\t"
172         "psrlq  $8, %%mm6\n\t"
173         "psrlq  $8, %%mm7\n\t"
174         "pand   %2, %%mm0\n\t"
175         "pand   %2, %%mm1\n\t"
176         "pand   %2, %%mm4\n\t"
177         "pand   %2, %%mm5\n\t"
178         "pand   %3, %%mm2\n\t"
179         "pand   %3, %%mm3\n\t"
180         "pand   %3, %%mm6\n\t"
181         "pand   %3, %%mm7\n\t"
182         "por    %%mm2, %%mm0\n\t"
183         "por    %%mm3, %%mm1\n\t"
184         "por    %%mm6, %%mm4\n\t"
185         "por    %%mm7, %%mm5\n\t"
186
187         "movq   %%mm1, %%mm2\n\t"
188         "movq   %%mm4, %%mm3\n\t"
189         "psllq  $48, %%mm2\n\t"
190         "psllq  $32, %%mm3\n\t"
191         "pand   %4, %%mm2\n\t"
192         "pand   %5, %%mm3\n\t"
193         "por    %%mm2, %%mm0\n\t"
194         "psrlq  $16, %%mm1\n\t"
195         "psrlq  $32, %%mm4\n\t"
196         "psllq  $16, %%mm5\n\t"
197         "por    %%mm3, %%mm1\n\t"
198         "pand   %6, %%mm5\n\t"
199         "por    %%mm5, %%mm4\n\t"
200
201         MOVNTQ" %%mm0, %0\n\t"
202         MOVNTQ" %%mm1, 8%0\n\t"
203         MOVNTQ" %%mm4, 16%0"
204         :"=m"(*dest)
205         :"m"(*s),"m"(mask24l),
206          "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
207         :"memory");
208     dest += 24;
209     s += 32;
210   }
211   __asm __volatile(SFENCE:::"memory");
212   __asm __volatile(EMMS:::"memory");
213 #endif
214   while(s < end)
215   {
216 #ifdef WORDS_BIGENDIAN
217     /* RGB32 (= A,B,G,R) -> RGB24 (= R,G,B) */
218     s++;
219     dest[2] = *s++;
220     dest[1] = *s++;
221     dest[0] = *s++;
222     dest += 3;
223 #else
224     *dest++ = *s++;
225     *dest++ = *s++;
226     *dest++ = *s++;
227     s++;
228 #endif
229   }
230 }
231
232 /*
233  Original by Strepto/Astral
234  ported to gcc & bugfixed : A'rpi
235  MMX2, 3DNOW optimization by Nick Kurshev
236  32bit c version, and and&add trick by Michael Niedermayer
237 */
238 static inline void RENAME(rgb15to16)(const uint8_t *src,uint8_t *dst,long src_size)
239 {
240   register const uint8_t* s=src;
241   register uint8_t* d=dst;
242   register const uint8_t *end;
243   const uint8_t *mm_end;
244   end = s + src_size;
245 #ifdef HAVE_MMX
246   __asm __volatile(PREFETCH"    %0"::"m"(*s));
247   __asm __volatile("movq        %0, %%mm4"::"m"(mask15s));
248   mm_end = end - 15;
249   while(s<mm_end)
250   {
251         __asm __volatile(
252                 PREFETCH"       32%1\n\t"
253                 "movq   %1, %%mm0\n\t"
254                 "movq   8%1, %%mm2\n\t"
255                 "movq   %%mm0, %%mm1\n\t"
256                 "movq   %%mm2, %%mm3\n\t"
257                 "pand   %%mm4, %%mm0\n\t"
258                 "pand   %%mm4, %%mm2\n\t"
259                 "paddw  %%mm1, %%mm0\n\t"
260                 "paddw  %%mm3, %%mm2\n\t"
261                 MOVNTQ" %%mm0, %0\n\t"
262                 MOVNTQ" %%mm2, 8%0"
263                 :"=m"(*d)
264                 :"m"(*s)
265                 );
266         d+=16;
267         s+=16;
268   }
269   __asm __volatile(SFENCE:::"memory");
270   __asm __volatile(EMMS:::"memory");
271 #endif
272     mm_end = end - 3;
273     while(s < mm_end)
274     {
275         register unsigned x= *((uint32_t *)s);
276         *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
277         d+=4;
278         s+=4;
279     }
280     if(s < end)
281     {
282         register unsigned short x= *((uint16_t *)s);
283         *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
284     }
285 }
286
287 static inline void RENAME(rgb16to15)(const uint8_t *src,uint8_t *dst,long src_size)
288 {
289   register const uint8_t* s=src;
290   register uint8_t* d=dst;
291   register const uint8_t *end;
292   const uint8_t *mm_end;
293   end = s + src_size;
294 #ifdef HAVE_MMX
295   __asm __volatile(PREFETCH"    %0"::"m"(*s));
296   __asm __volatile("movq        %0, %%mm7"::"m"(mask15rg));
297   __asm __volatile("movq        %0, %%mm6"::"m"(mask15b));
298   mm_end = end - 15;
299   while(s<mm_end)
300   {
301         __asm __volatile(
302                 PREFETCH"       32%1\n\t"
303                 "movq   %1, %%mm0\n\t"
304                 "movq   8%1, %%mm2\n\t"
305                 "movq   %%mm0, %%mm1\n\t"
306                 "movq   %%mm2, %%mm3\n\t"
307                 "psrlq  $1, %%mm0\n\t"
308                 "psrlq  $1, %%mm2\n\t"
309                 "pand   %%mm7, %%mm0\n\t"
310                 "pand   %%mm7, %%mm2\n\t"
311                 "pand   %%mm6, %%mm1\n\t"
312                 "pand   %%mm6, %%mm3\n\t"
313                 "por    %%mm1, %%mm0\n\t"
314                 "por    %%mm3, %%mm2\n\t"
315                 MOVNTQ" %%mm0, %0\n\t"
316                 MOVNTQ" %%mm2, 8%0"
317                 :"=m"(*d)
318                 :"m"(*s)
319                 );
320         d+=16;
321         s+=16;
322   }
323   __asm __volatile(SFENCE:::"memory");
324   __asm __volatile(EMMS:::"memory");
325 #endif
326     mm_end = end - 3;
327     while(s < mm_end)
328     {
329         register uint32_t x= *((uint32_t *)s);
330         *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
331         s+=4;
332         d+=4;
333     }
334     if(s < end)
335     {
336         register uint16_t x= *((uint16_t *)s);
337         *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
338         s+=2;
339         d+=2;
340     }
341 }
342
343 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, long src_size)
344 {
345         const uint8_t *s = src;
346         const uint8_t *end;
347 #ifdef HAVE_MMX
348         const uint8_t *mm_end;
349 #endif
350         uint16_t *d = (uint16_t *)dst;
351         end = s + src_size;
352 #ifdef HAVE_MMX
353         mm_end = end - 15;
354 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
355         asm volatile(
356                 "movq %3, %%mm5                 \n\t"
357                 "movq %4, %%mm6                 \n\t"
358                 "movq %5, %%mm7                 \n\t"
359                 "jmp 2f                         \n\t"
360                 ASMALIGN(4)
361                 "1:                             \n\t"
362                 PREFETCH" 32(%1)                \n\t"
363                 "movd   (%1), %%mm0             \n\t"
364                 "movd   4(%1), %%mm3            \n\t"
365                 "punpckldq 8(%1), %%mm0         \n\t"
366                 "punpckldq 12(%1), %%mm3        \n\t"
367                 "movq %%mm0, %%mm1              \n\t"
368                 "movq %%mm3, %%mm4              \n\t"
369                 "pand %%mm6, %%mm0              \n\t"
370                 "pand %%mm6, %%mm3              \n\t"
371                 "pmaddwd %%mm7, %%mm0           \n\t"
372                 "pmaddwd %%mm7, %%mm3           \n\t"
373                 "pand %%mm5, %%mm1              \n\t"
374                 "pand %%mm5, %%mm4              \n\t"
375                 "por %%mm1, %%mm0               \n\t"   
376                 "por %%mm4, %%mm3               \n\t"
377                 "psrld $5, %%mm0                \n\t"
378                 "pslld $11, %%mm3               \n\t"
379                 "por %%mm3, %%mm0               \n\t"
380                 MOVNTQ" %%mm0, (%0)             \n\t"
381                 "add $16, %1                    \n\t"
382                 "add $8, %0                     \n\t"
383                 "2:                             \n\t"
384                 "cmp %2, %1                     \n\t"
385                 " jb 1b                         \n\t"
386                 : "+r" (d), "+r"(s)
387                 : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
388         );
389 #else
390         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
391         __asm __volatile(
392             "movq       %0, %%mm7\n\t"
393             "movq       %1, %%mm6\n\t"
394             ::"m"(red_16mask),"m"(green_16mask));
395         while(s < mm_end)
396         {
397             __asm __volatile(
398                 PREFETCH" 32%1\n\t"
399                 "movd   %1, %%mm0\n\t"
400                 "movd   4%1, %%mm3\n\t"
401                 "punpckldq 8%1, %%mm0\n\t"
402                 "punpckldq 12%1, %%mm3\n\t"
403                 "movq   %%mm0, %%mm1\n\t"
404                 "movq   %%mm0, %%mm2\n\t"
405                 "movq   %%mm3, %%mm4\n\t"
406                 "movq   %%mm3, %%mm5\n\t"
407                 "psrlq  $3, %%mm0\n\t"
408                 "psrlq  $3, %%mm3\n\t"
409                 "pand   %2, %%mm0\n\t"
410                 "pand   %2, %%mm3\n\t"
411                 "psrlq  $5, %%mm1\n\t"
412                 "psrlq  $5, %%mm4\n\t"
413                 "pand   %%mm6, %%mm1\n\t"
414                 "pand   %%mm6, %%mm4\n\t"
415                 "psrlq  $8, %%mm2\n\t"
416                 "psrlq  $8, %%mm5\n\t"
417                 "pand   %%mm7, %%mm2\n\t"
418                 "pand   %%mm7, %%mm5\n\t"
419                 "por    %%mm1, %%mm0\n\t"
420                 "por    %%mm4, %%mm3\n\t"
421                 "por    %%mm2, %%mm0\n\t"
422                 "por    %%mm5, %%mm3\n\t"
423                 "psllq  $16, %%mm3\n\t"
424                 "por    %%mm3, %%mm0\n\t"
425                 MOVNTQ" %%mm0, %0\n\t"
426                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
427                 d += 4;
428                 s += 16;
429         }
430 #endif
431         __asm __volatile(SFENCE:::"memory");
432         __asm __volatile(EMMS:::"memory");
433 #endif
434         while(s < end)
435         {
436                 register int rgb = *(uint32_t*)s; s += 4;
437                 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
438         }
439 }
440
441 static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
442 {
443         const uint8_t *s = src;
444         const uint8_t *end;
445 #ifdef HAVE_MMX
446         const uint8_t *mm_end;
447 #endif
448         uint16_t *d = (uint16_t *)dst;
449         end = s + src_size;
450 #ifdef HAVE_MMX
451         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
452         __asm __volatile(
453             "movq       %0, %%mm7\n\t"
454             "movq       %1, %%mm6\n\t"
455             ::"m"(red_16mask),"m"(green_16mask));
456         mm_end = end - 15;
457         while(s < mm_end)
458         {
459             __asm __volatile(
460                 PREFETCH" 32%1\n\t"
461                 "movd   %1, %%mm0\n\t"
462                 "movd   4%1, %%mm3\n\t"
463                 "punpckldq 8%1, %%mm0\n\t"
464                 "punpckldq 12%1, %%mm3\n\t"
465                 "movq   %%mm0, %%mm1\n\t"
466                 "movq   %%mm0, %%mm2\n\t"
467                 "movq   %%mm3, %%mm4\n\t"
468                 "movq   %%mm3, %%mm5\n\t"
469                 "psllq  $8, %%mm0\n\t"
470                 "psllq  $8, %%mm3\n\t"
471                 "pand   %%mm7, %%mm0\n\t"
472                 "pand   %%mm7, %%mm3\n\t"
473                 "psrlq  $5, %%mm1\n\t"
474                 "psrlq  $5, %%mm4\n\t"
475                 "pand   %%mm6, %%mm1\n\t"
476                 "pand   %%mm6, %%mm4\n\t"
477                 "psrlq  $19, %%mm2\n\t"
478                 "psrlq  $19, %%mm5\n\t"
479                 "pand   %2, %%mm2\n\t"
480                 "pand   %2, %%mm5\n\t"
481                 "por    %%mm1, %%mm0\n\t"
482                 "por    %%mm4, %%mm3\n\t"
483                 "por    %%mm2, %%mm0\n\t"
484                 "por    %%mm5, %%mm3\n\t"
485                 "psllq  $16, %%mm3\n\t"
486                 "por    %%mm3, %%mm0\n\t"
487                 MOVNTQ" %%mm0, %0\n\t"
488                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
489                 d += 4;
490                 s += 16;
491         }
492         __asm __volatile(SFENCE:::"memory");
493         __asm __volatile(EMMS:::"memory");
494 #endif
495         while(s < end)
496         {
497                 register int rgb = *(uint32_t*)s; s += 4;
498                 *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
499         }
500 }
501
502 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, long src_size)
503 {
504         const uint8_t *s = src;
505         const uint8_t *end;
506 #ifdef HAVE_MMX
507         const uint8_t *mm_end;
508 #endif
509         uint16_t *d = (uint16_t *)dst;
510         end = s + src_size;
511 #ifdef HAVE_MMX
512         mm_end = end - 15;
513 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
514         asm volatile(
515                 "movq %3, %%mm5                 \n\t"
516                 "movq %4, %%mm6                 \n\t"
517                 "movq %5, %%mm7                 \n\t"
518                 "jmp 2f                         \n\t"
519                 ASMALIGN(4)
520                 "1:                             \n\t"
521                 PREFETCH" 32(%1)                \n\t"
522                 "movd   (%1), %%mm0             \n\t"
523                 "movd   4(%1), %%mm3            \n\t"
524                 "punpckldq 8(%1), %%mm0         \n\t"
525                 "punpckldq 12(%1), %%mm3        \n\t"
526                 "movq %%mm0, %%mm1              \n\t"
527                 "movq %%mm3, %%mm4              \n\t"
528                 "pand %%mm6, %%mm0              \n\t"
529                 "pand %%mm6, %%mm3              \n\t"
530                 "pmaddwd %%mm7, %%mm0           \n\t"
531                 "pmaddwd %%mm7, %%mm3           \n\t"
532                 "pand %%mm5, %%mm1              \n\t"
533                 "pand %%mm5, %%mm4              \n\t"
534                 "por %%mm1, %%mm0               \n\t"   
535                 "por %%mm4, %%mm3               \n\t"
536                 "psrld $6, %%mm0                \n\t"
537                 "pslld $10, %%mm3               \n\t"
538                 "por %%mm3, %%mm0               \n\t"
539                 MOVNTQ" %%mm0, (%0)             \n\t"
540                 "add $16, %1                    \n\t"
541                 "add $8, %0                     \n\t"
542                 "2:                             \n\t"
543                 "cmp %2, %1                     \n\t"
544                 " jb 1b                         \n\t"
545                 : "+r" (d), "+r"(s)
546                 : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
547         );
548 #else
549         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
550         __asm __volatile(
551             "movq       %0, %%mm7\n\t"
552             "movq       %1, %%mm6\n\t"
553             ::"m"(red_15mask),"m"(green_15mask));
554         while(s < mm_end)
555         {
556             __asm __volatile(
557                 PREFETCH" 32%1\n\t"
558                 "movd   %1, %%mm0\n\t"
559                 "movd   4%1, %%mm3\n\t"
560                 "punpckldq 8%1, %%mm0\n\t"
561                 "punpckldq 12%1, %%mm3\n\t"
562                 "movq   %%mm0, %%mm1\n\t"
563                 "movq   %%mm0, %%mm2\n\t"
564                 "movq   %%mm3, %%mm4\n\t"
565                 "movq   %%mm3, %%mm5\n\t"
566                 "psrlq  $3, %%mm0\n\t"
567                 "psrlq  $3, %%mm3\n\t"
568                 "pand   %2, %%mm0\n\t"
569                 "pand   %2, %%mm3\n\t"
570                 "psrlq  $6, %%mm1\n\t"
571                 "psrlq  $6, %%mm4\n\t"
572                 "pand   %%mm6, %%mm1\n\t"
573                 "pand   %%mm6, %%mm4\n\t"
574                 "psrlq  $9, %%mm2\n\t"
575                 "psrlq  $9, %%mm5\n\t"
576                 "pand   %%mm7, %%mm2\n\t"
577                 "pand   %%mm7, %%mm5\n\t"
578                 "por    %%mm1, %%mm0\n\t"
579                 "por    %%mm4, %%mm3\n\t"
580                 "por    %%mm2, %%mm0\n\t"
581                 "por    %%mm5, %%mm3\n\t"
582                 "psllq  $16, %%mm3\n\t"
583                 "por    %%mm3, %%mm0\n\t"
584                 MOVNTQ" %%mm0, %0\n\t"
585                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
586                 d += 4;
587                 s += 16;
588         }
589 #endif
590         __asm __volatile(SFENCE:::"memory");
591         __asm __volatile(EMMS:::"memory");
592 #endif
593         while(s < end)
594         {
595                 register int rgb = *(uint32_t*)s; s += 4;
596                 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
597         }
598 }
599
600 static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
601 {
602         const uint8_t *s = src;
603         const uint8_t *end;
604 #ifdef HAVE_MMX
605         const uint8_t *mm_end;
606 #endif
607         uint16_t *d = (uint16_t *)dst;
608         end = s + src_size;
609 #ifdef HAVE_MMX
610         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
611         __asm __volatile(
612             "movq       %0, %%mm7\n\t"
613             "movq       %1, %%mm6\n\t"
614             ::"m"(red_15mask),"m"(green_15mask));
615         mm_end = end - 15;
616         while(s < mm_end)
617         {
618             __asm __volatile(
619                 PREFETCH" 32%1\n\t"
620                 "movd   %1, %%mm0\n\t"
621                 "movd   4%1, %%mm3\n\t"
622                 "punpckldq 8%1, %%mm0\n\t"
623                 "punpckldq 12%1, %%mm3\n\t"
624                 "movq   %%mm0, %%mm1\n\t"
625                 "movq   %%mm0, %%mm2\n\t"
626                 "movq   %%mm3, %%mm4\n\t"
627                 "movq   %%mm3, %%mm5\n\t"
628                 "psllq  $7, %%mm0\n\t"
629                 "psllq  $7, %%mm3\n\t"
630                 "pand   %%mm7, %%mm0\n\t"
631                 "pand   %%mm7, %%mm3\n\t"
632                 "psrlq  $6, %%mm1\n\t"
633                 "psrlq  $6, %%mm4\n\t"
634                 "pand   %%mm6, %%mm1\n\t"
635                 "pand   %%mm6, %%mm4\n\t"
636                 "psrlq  $19, %%mm2\n\t"
637                 "psrlq  $19, %%mm5\n\t"
638                 "pand   %2, %%mm2\n\t"
639                 "pand   %2, %%mm5\n\t"
640                 "por    %%mm1, %%mm0\n\t"
641                 "por    %%mm4, %%mm3\n\t"
642                 "por    %%mm2, %%mm0\n\t"
643                 "por    %%mm5, %%mm3\n\t"
644                 "psllq  $16, %%mm3\n\t"
645                 "por    %%mm3, %%mm0\n\t"
646                 MOVNTQ" %%mm0, %0\n\t"
647                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
648                 d += 4;
649                 s += 16;
650         }
651         __asm __volatile(SFENCE:::"memory");
652         __asm __volatile(EMMS:::"memory");
653 #endif
654         while(s < end)
655         {
656                 register int rgb = *(uint32_t*)s; s += 4;
657                 *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
658         }
659 }
660
661 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, long src_size)
662 {
663         const uint8_t *s = src;
664         const uint8_t *end;
665 #ifdef HAVE_MMX
666         const uint8_t *mm_end;
667 #endif
668         uint16_t *d = (uint16_t *)dst;
669         end = s + src_size;
670 #ifdef HAVE_MMX
671         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
672         __asm __volatile(
673             "movq       %0, %%mm7\n\t"
674             "movq       %1, %%mm6\n\t"
675             ::"m"(red_16mask),"m"(green_16mask));
676         mm_end = end - 11;
677         while(s < mm_end)
678         {
679             __asm __volatile(
680                 PREFETCH" 32%1\n\t"
681                 "movd   %1, %%mm0\n\t"
682                 "movd   3%1, %%mm3\n\t"
683                 "punpckldq 6%1, %%mm0\n\t"
684                 "punpckldq 9%1, %%mm3\n\t"
685                 "movq   %%mm0, %%mm1\n\t"
686                 "movq   %%mm0, %%mm2\n\t"
687                 "movq   %%mm3, %%mm4\n\t"
688                 "movq   %%mm3, %%mm5\n\t"
689                 "psrlq  $3, %%mm0\n\t"
690                 "psrlq  $3, %%mm3\n\t"
691                 "pand   %2, %%mm0\n\t"
692                 "pand   %2, %%mm3\n\t"
693                 "psrlq  $5, %%mm1\n\t"
694                 "psrlq  $5, %%mm4\n\t"
695                 "pand   %%mm6, %%mm1\n\t"
696                 "pand   %%mm6, %%mm4\n\t"
697                 "psrlq  $8, %%mm2\n\t"
698                 "psrlq  $8, %%mm5\n\t"
699                 "pand   %%mm7, %%mm2\n\t"
700                 "pand   %%mm7, %%mm5\n\t"
701                 "por    %%mm1, %%mm0\n\t"
702                 "por    %%mm4, %%mm3\n\t"
703                 "por    %%mm2, %%mm0\n\t"
704                 "por    %%mm5, %%mm3\n\t"
705                 "psllq  $16, %%mm3\n\t"
706                 "por    %%mm3, %%mm0\n\t"
707                 MOVNTQ" %%mm0, %0\n\t"
708                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
709                 d += 4;
710                 s += 12;
711         }
712         __asm __volatile(SFENCE:::"memory");
713         __asm __volatile(EMMS:::"memory");
714 #endif
715         while(s < end)
716         {
717                 const int b= *s++;
718                 const int g= *s++;
719                 const int r= *s++;
720                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
721         }
722 }
723
724 static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
725 {
726         const uint8_t *s = src;
727         const uint8_t *end;
728 #ifdef HAVE_MMX
729         const uint8_t *mm_end;
730 #endif
731         uint16_t *d = (uint16_t *)dst;
732         end = s + src_size;
733 #ifdef HAVE_MMX
734         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
735         __asm __volatile(
736             "movq       %0, %%mm7\n\t"
737             "movq       %1, %%mm6\n\t"
738             ::"m"(red_16mask),"m"(green_16mask));
739         mm_end = end - 15;
740         while(s < mm_end)
741         {
742             __asm __volatile(
743                 PREFETCH" 32%1\n\t"
744                 "movd   %1, %%mm0\n\t"
745                 "movd   3%1, %%mm3\n\t"
746                 "punpckldq 6%1, %%mm0\n\t"
747                 "punpckldq 9%1, %%mm3\n\t"
748                 "movq   %%mm0, %%mm1\n\t"
749                 "movq   %%mm0, %%mm2\n\t"
750                 "movq   %%mm3, %%mm4\n\t"
751                 "movq   %%mm3, %%mm5\n\t"
752                 "psllq  $8, %%mm0\n\t"
753                 "psllq  $8, %%mm3\n\t"
754                 "pand   %%mm7, %%mm0\n\t"
755                 "pand   %%mm7, %%mm3\n\t"
756                 "psrlq  $5, %%mm1\n\t"
757                 "psrlq  $5, %%mm4\n\t"
758                 "pand   %%mm6, %%mm1\n\t"
759                 "pand   %%mm6, %%mm4\n\t"
760                 "psrlq  $19, %%mm2\n\t"
761                 "psrlq  $19, %%mm5\n\t"
762                 "pand   %2, %%mm2\n\t"
763                 "pand   %2, %%mm5\n\t"
764                 "por    %%mm1, %%mm0\n\t"
765                 "por    %%mm4, %%mm3\n\t"
766                 "por    %%mm2, %%mm0\n\t"
767                 "por    %%mm5, %%mm3\n\t"
768                 "psllq  $16, %%mm3\n\t"
769                 "por    %%mm3, %%mm0\n\t"
770                 MOVNTQ" %%mm0, %0\n\t"
771                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
772                 d += 4;
773                 s += 12;
774         }
775         __asm __volatile(SFENCE:::"memory");
776         __asm __volatile(EMMS:::"memory");
777 #endif
778         while(s < end)
779         {
780                 const int r= *s++;
781                 const int g= *s++;
782                 const int b= *s++;
783                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
784         }
785 }
786
787 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, long src_size)
788 {
789         const uint8_t *s = src;
790         const uint8_t *end;
791 #ifdef HAVE_MMX
792         const uint8_t *mm_end;
793 #endif
794         uint16_t *d = (uint16_t *)dst;
795         end = s + src_size;
796 #ifdef HAVE_MMX
797         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
798         __asm __volatile(
799             "movq       %0, %%mm7\n\t"
800             "movq       %1, %%mm6\n\t"
801             ::"m"(red_15mask),"m"(green_15mask));
802         mm_end = end - 11;
803         while(s < mm_end)
804         {
805             __asm __volatile(
806                 PREFETCH" 32%1\n\t"
807                 "movd   %1, %%mm0\n\t"
808                 "movd   3%1, %%mm3\n\t"
809                 "punpckldq 6%1, %%mm0\n\t"
810                 "punpckldq 9%1, %%mm3\n\t"
811                 "movq   %%mm0, %%mm1\n\t"
812                 "movq   %%mm0, %%mm2\n\t"
813                 "movq   %%mm3, %%mm4\n\t"
814                 "movq   %%mm3, %%mm5\n\t"
815                 "psrlq  $3, %%mm0\n\t"
816                 "psrlq  $3, %%mm3\n\t"
817                 "pand   %2, %%mm0\n\t"
818                 "pand   %2, %%mm3\n\t"
819                 "psrlq  $6, %%mm1\n\t"
820                 "psrlq  $6, %%mm4\n\t"
821                 "pand   %%mm6, %%mm1\n\t"
822                 "pand   %%mm6, %%mm4\n\t"
823                 "psrlq  $9, %%mm2\n\t"
824                 "psrlq  $9, %%mm5\n\t"
825                 "pand   %%mm7, %%mm2\n\t"
826                 "pand   %%mm7, %%mm5\n\t"
827                 "por    %%mm1, %%mm0\n\t"
828                 "por    %%mm4, %%mm3\n\t"
829                 "por    %%mm2, %%mm0\n\t"
830                 "por    %%mm5, %%mm3\n\t"
831                 "psllq  $16, %%mm3\n\t"
832                 "por    %%mm3, %%mm0\n\t"
833                 MOVNTQ" %%mm0, %0\n\t"
834                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
835                 d += 4;
836                 s += 12;
837         }
838         __asm __volatile(SFENCE:::"memory");
839         __asm __volatile(EMMS:::"memory");
840 #endif
841         while(s < end)
842         {
843                 const int b= *s++;
844                 const int g= *s++;
845                 const int r= *s++;
846                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
847         }
848 }
849
850 static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
851 {
852         const uint8_t *s = src;
853         const uint8_t *end;
854 #ifdef HAVE_MMX
855         const uint8_t *mm_end;
856 #endif
857         uint16_t *d = (uint16_t *)dst;
858         end = s + src_size;
859 #ifdef HAVE_MMX
860         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
861         __asm __volatile(
862             "movq       %0, %%mm7\n\t"
863             "movq       %1, %%mm6\n\t"
864             ::"m"(red_15mask),"m"(green_15mask));
865         mm_end = end - 15;
866         while(s < mm_end)
867         {
868             __asm __volatile(
869                 PREFETCH" 32%1\n\t"
870                 "movd   %1, %%mm0\n\t"
871                 "movd   3%1, %%mm3\n\t"
872                 "punpckldq 6%1, %%mm0\n\t"
873                 "punpckldq 9%1, %%mm3\n\t"
874                 "movq   %%mm0, %%mm1\n\t"
875                 "movq   %%mm0, %%mm2\n\t"
876                 "movq   %%mm3, %%mm4\n\t"
877                 "movq   %%mm3, %%mm5\n\t"
878                 "psllq  $7, %%mm0\n\t"
879                 "psllq  $7, %%mm3\n\t"
880                 "pand   %%mm7, %%mm0\n\t"
881                 "pand   %%mm7, %%mm3\n\t"
882                 "psrlq  $6, %%mm1\n\t"
883                 "psrlq  $6, %%mm4\n\t"
884                 "pand   %%mm6, %%mm1\n\t"
885                 "pand   %%mm6, %%mm4\n\t"
886                 "psrlq  $19, %%mm2\n\t"
887                 "psrlq  $19, %%mm5\n\t"
888                 "pand   %2, %%mm2\n\t"
889                 "pand   %2, %%mm5\n\t"
890                 "por    %%mm1, %%mm0\n\t"
891                 "por    %%mm4, %%mm3\n\t"
892                 "por    %%mm2, %%mm0\n\t"
893                 "por    %%mm5, %%mm3\n\t"
894                 "psllq  $16, %%mm3\n\t"
895                 "por    %%mm3, %%mm0\n\t"
896                 MOVNTQ" %%mm0, %0\n\t"
897                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
898                 d += 4;
899                 s += 12;
900         }
901         __asm __volatile(SFENCE:::"memory");
902         __asm __volatile(EMMS:::"memory");
903 #endif
904         while(s < end)
905         {
906                 const int r= *s++;
907                 const int g= *s++;
908                 const int b= *s++;
909                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
910         }
911 }
912
913 /*
914   I use here less accurate approximation by simply
915  left-shifting the input
916   value and filling the low order bits with
917  zeroes. This method improves png's
918   compression but this scheme cannot reproduce white exactly, since it does not
919   generate an all-ones maximum value; the net effect is to darken the
920   image slightly.
921
922   The better method should be "left bit replication":
923
924    4 3 2 1 0
925    ---------
926    1 1 0 1 1
927
928    7 6 5 4 3  2 1 0
929    ----------------
930    1 1 0 1 1  1 1 0
931    |=======|  |===|
932        |      Leftmost Bits Repeated to Fill Open Bits
933        |
934    Original Bits
935 */
936 static inline void RENAME(rgb15to24)(const uint8_t *src, uint8_t *dst, long src_size)
937 {
938         const uint16_t *end;
939 #ifdef HAVE_MMX
940         const uint16_t *mm_end;
941 #endif
942         uint8_t *d = (uint8_t *)dst;
943         const uint16_t *s = (uint16_t *)src;
944         end = s + src_size/2;
945 #ifdef HAVE_MMX
946         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
947         mm_end = end - 7;
948         while(s < mm_end)
949         {
950             __asm __volatile(
951                 PREFETCH" 32%1\n\t"
952                 "movq   %1, %%mm0\n\t"
953                 "movq   %1, %%mm1\n\t"
954                 "movq   %1, %%mm2\n\t"
955                 "pand   %2, %%mm0\n\t"
956                 "pand   %3, %%mm1\n\t"
957                 "pand   %4, %%mm2\n\t"
958                 "psllq  $3, %%mm0\n\t"
959                 "psrlq  $2, %%mm1\n\t"
960                 "psrlq  $7, %%mm2\n\t"
961                 "movq   %%mm0, %%mm3\n\t"
962                 "movq   %%mm1, %%mm4\n\t"
963                 "movq   %%mm2, %%mm5\n\t"
964                 "punpcklwd %5, %%mm0\n\t"
965                 "punpcklwd %5, %%mm1\n\t"
966                 "punpcklwd %5, %%mm2\n\t"
967                 "punpckhwd %5, %%mm3\n\t"
968                 "punpckhwd %5, %%mm4\n\t"
969                 "punpckhwd %5, %%mm5\n\t"
970                 "psllq  $8, %%mm1\n\t"
971                 "psllq  $16, %%mm2\n\t"
972                 "por    %%mm1, %%mm0\n\t"
973                 "por    %%mm2, %%mm0\n\t"
974                 "psllq  $8, %%mm4\n\t"
975                 "psllq  $16, %%mm5\n\t"
976                 "por    %%mm4, %%mm3\n\t"
977                 "por    %%mm5, %%mm3\n\t"
978
979                 "movq   %%mm0, %%mm6\n\t"
980                 "movq   %%mm3, %%mm7\n\t"
981                 
982                 "movq   8%1, %%mm0\n\t"
983                 "movq   8%1, %%mm1\n\t"
984                 "movq   8%1, %%mm2\n\t"
985                 "pand   %2, %%mm0\n\t"
986                 "pand   %3, %%mm1\n\t"
987                 "pand   %4, %%mm2\n\t"
988                 "psllq  $3, %%mm0\n\t"
989                 "psrlq  $2, %%mm1\n\t"
990                 "psrlq  $7, %%mm2\n\t"
991                 "movq   %%mm0, %%mm3\n\t"
992                 "movq   %%mm1, %%mm4\n\t"
993                 "movq   %%mm2, %%mm5\n\t"
994                 "punpcklwd %5, %%mm0\n\t"
995                 "punpcklwd %5, %%mm1\n\t"
996                 "punpcklwd %5, %%mm2\n\t"
997                 "punpckhwd %5, %%mm3\n\t"
998                 "punpckhwd %5, %%mm4\n\t"
999                 "punpckhwd %5, %%mm5\n\t"
1000                 "psllq  $8, %%mm1\n\t"
1001                 "psllq  $16, %%mm2\n\t"
1002                 "por    %%mm1, %%mm0\n\t"
1003                 "por    %%mm2, %%mm0\n\t"
1004                 "psllq  $8, %%mm4\n\t"
1005                 "psllq  $16, %%mm5\n\t"
1006                 "por    %%mm4, %%mm3\n\t"
1007                 "por    %%mm5, %%mm3\n\t"
1008
1009                 :"=m"(*d)
1010                 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)
1011                 :"memory");
1012             /* Borrowed 32 to 24 */
1013             __asm __volatile(
1014                 "movq   %%mm0, %%mm4\n\t"
1015                 "movq   %%mm3, %%mm5\n\t"
1016                 "movq   %%mm6, %%mm0\n\t"
1017                 "movq   %%mm7, %%mm1\n\t"
1018                 
1019                 "movq   %%mm4, %%mm6\n\t"
1020                 "movq   %%mm5, %%mm7\n\t"
1021                 "movq   %%mm0, %%mm2\n\t"
1022                 "movq   %%mm1, %%mm3\n\t"
1023
1024                 "psrlq  $8, %%mm2\n\t"
1025                 "psrlq  $8, %%mm3\n\t"
1026                 "psrlq  $8, %%mm6\n\t"
1027                 "psrlq  $8, %%mm7\n\t"
1028                 "pand   %2, %%mm0\n\t"
1029                 "pand   %2, %%mm1\n\t"
1030                 "pand   %2, %%mm4\n\t"
1031                 "pand   %2, %%mm5\n\t"
1032                 "pand   %3, %%mm2\n\t"
1033                 "pand   %3, %%mm3\n\t"
1034                 "pand   %3, %%mm6\n\t"
1035                 "pand   %3, %%mm7\n\t"
1036                 "por    %%mm2, %%mm0\n\t"
1037                 "por    %%mm3, %%mm1\n\t"
1038                 "por    %%mm6, %%mm4\n\t"
1039                 "por    %%mm7, %%mm5\n\t"
1040
1041                 "movq   %%mm1, %%mm2\n\t"
1042                 "movq   %%mm4, %%mm3\n\t"
1043                 "psllq  $48, %%mm2\n\t"
1044                 "psllq  $32, %%mm3\n\t"
1045                 "pand   %4, %%mm2\n\t"
1046                 "pand   %5, %%mm3\n\t"
1047                 "por    %%mm2, %%mm0\n\t"
1048                 "psrlq  $16, %%mm1\n\t"
1049                 "psrlq  $32, %%mm4\n\t"
1050                 "psllq  $16, %%mm5\n\t"
1051                 "por    %%mm3, %%mm1\n\t"
1052                 "pand   %6, %%mm5\n\t"
1053                 "por    %%mm5, %%mm4\n\t"
1054
1055                 MOVNTQ" %%mm0, %0\n\t"
1056                 MOVNTQ" %%mm1, 8%0\n\t"
1057                 MOVNTQ" %%mm4, 16%0"
1058
1059                 :"=m"(*d)
1060                 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1061                 :"memory");
1062                 d += 24;
1063                 s += 8;
1064         }
1065         __asm __volatile(SFENCE:::"memory");
1066         __asm __volatile(EMMS:::"memory");
1067 #endif
1068         while(s < end)
1069         {
1070                 register uint16_t bgr;
1071                 bgr = *s++;
1072                 *d++ = (bgr&0x1F)<<3;
1073                 *d++ = (bgr&0x3E0)>>2;
1074                 *d++ = (bgr&0x7C00)>>7;
1075         }
1076 }
1077
1078 static inline void RENAME(rgb16to24)(const uint8_t *src, uint8_t *dst, long src_size)
1079 {
1080         const uint16_t *end;
1081 #ifdef HAVE_MMX
1082         const uint16_t *mm_end;
1083 #endif
1084         uint8_t *d = (uint8_t *)dst;
1085         const uint16_t *s = (const uint16_t *)src;
1086         end = s + src_size/2;
1087 #ifdef HAVE_MMX
1088         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1089         mm_end = end - 7;
1090         while(s < mm_end)
1091         {
1092             __asm __volatile(
1093                 PREFETCH" 32%1\n\t"
1094                 "movq   %1, %%mm0\n\t"
1095                 "movq   %1, %%mm1\n\t"
1096                 "movq   %1, %%mm2\n\t"
1097                 "pand   %2, %%mm0\n\t"
1098                 "pand   %3, %%mm1\n\t"
1099                 "pand   %4, %%mm2\n\t"
1100                 "psllq  $3, %%mm0\n\t"
1101                 "psrlq  $3, %%mm1\n\t"
1102                 "psrlq  $8, %%mm2\n\t"
1103                 "movq   %%mm0, %%mm3\n\t"
1104                 "movq   %%mm1, %%mm4\n\t"
1105                 "movq   %%mm2, %%mm5\n\t"
1106                 "punpcklwd %5, %%mm0\n\t"
1107                 "punpcklwd %5, %%mm1\n\t"
1108                 "punpcklwd %5, %%mm2\n\t"
1109                 "punpckhwd %5, %%mm3\n\t"
1110                 "punpckhwd %5, %%mm4\n\t"
1111                 "punpckhwd %5, %%mm5\n\t"
1112                 "psllq  $8, %%mm1\n\t"
1113                 "psllq  $16, %%mm2\n\t"
1114                 "por    %%mm1, %%mm0\n\t"
1115                 "por    %%mm2, %%mm0\n\t"
1116                 "psllq  $8, %%mm4\n\t"
1117                 "psllq  $16, %%mm5\n\t"
1118                 "por    %%mm4, %%mm3\n\t"
1119                 "por    %%mm5, %%mm3\n\t"
1120                 
1121                 "movq   %%mm0, %%mm6\n\t"
1122                 "movq   %%mm3, %%mm7\n\t"
1123
1124                 "movq   8%1, %%mm0\n\t"
1125                 "movq   8%1, %%mm1\n\t"
1126                 "movq   8%1, %%mm2\n\t"
1127                 "pand   %2, %%mm0\n\t"
1128                 "pand   %3, %%mm1\n\t"
1129                 "pand   %4, %%mm2\n\t"
1130                 "psllq  $3, %%mm0\n\t"
1131                 "psrlq  $3, %%mm1\n\t"
1132                 "psrlq  $8, %%mm2\n\t"
1133                 "movq   %%mm0, %%mm3\n\t"
1134                 "movq   %%mm1, %%mm4\n\t"
1135                 "movq   %%mm2, %%mm5\n\t"
1136                 "punpcklwd %5, %%mm0\n\t"
1137                 "punpcklwd %5, %%mm1\n\t"
1138                 "punpcklwd %5, %%mm2\n\t"
1139                 "punpckhwd %5, %%mm3\n\t"
1140                 "punpckhwd %5, %%mm4\n\t"
1141                 "punpckhwd %5, %%mm5\n\t"
1142                 "psllq  $8, %%mm1\n\t"
1143                 "psllq  $16, %%mm2\n\t"
1144                 "por    %%mm1, %%mm0\n\t"
1145                 "por    %%mm2, %%mm0\n\t"
1146                 "psllq  $8, %%mm4\n\t"
1147                 "psllq  $16, %%mm5\n\t"
1148                 "por    %%mm4, %%mm3\n\t"
1149                 "por    %%mm5, %%mm3\n\t"
1150                 :"=m"(*d)
1151                 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)           
1152                 :"memory");
1153             /* Borrowed 32 to 24 */
1154             __asm __volatile(
1155                 "movq   %%mm0, %%mm4\n\t"
1156                 "movq   %%mm3, %%mm5\n\t"
1157                 "movq   %%mm6, %%mm0\n\t"
1158                 "movq   %%mm7, %%mm1\n\t"
1159                 
1160                 "movq   %%mm4, %%mm6\n\t"
1161                 "movq   %%mm5, %%mm7\n\t"
1162                 "movq   %%mm0, %%mm2\n\t"
1163                 "movq   %%mm1, %%mm3\n\t"
1164
1165                 "psrlq  $8, %%mm2\n\t"
1166                 "psrlq  $8, %%mm3\n\t"
1167                 "psrlq  $8, %%mm6\n\t"
1168                 "psrlq  $8, %%mm7\n\t"
1169                 "pand   %2, %%mm0\n\t"
1170                 "pand   %2, %%mm1\n\t"
1171                 "pand   %2, %%mm4\n\t"
1172                 "pand   %2, %%mm5\n\t"
1173                 "pand   %3, %%mm2\n\t"
1174                 "pand   %3, %%mm3\n\t"
1175                 "pand   %3, %%mm6\n\t"
1176                 "pand   %3, %%mm7\n\t"
1177                 "por    %%mm2, %%mm0\n\t"
1178                 "por    %%mm3, %%mm1\n\t"
1179                 "por    %%mm6, %%mm4\n\t"
1180                 "por    %%mm7, %%mm5\n\t"
1181
1182                 "movq   %%mm1, %%mm2\n\t"
1183                 "movq   %%mm4, %%mm3\n\t"
1184                 "psllq  $48, %%mm2\n\t"
1185                 "psllq  $32, %%mm3\n\t"
1186                 "pand   %4, %%mm2\n\t"
1187                 "pand   %5, %%mm3\n\t"
1188                 "por    %%mm2, %%mm0\n\t"
1189                 "psrlq  $16, %%mm1\n\t"
1190                 "psrlq  $32, %%mm4\n\t"
1191                 "psllq  $16, %%mm5\n\t"
1192                 "por    %%mm3, %%mm1\n\t"
1193                 "pand   %6, %%mm5\n\t"
1194                 "por    %%mm5, %%mm4\n\t"
1195
1196                 MOVNTQ" %%mm0, %0\n\t"
1197                 MOVNTQ" %%mm1, 8%0\n\t"
1198                 MOVNTQ" %%mm4, 16%0"
1199
1200                 :"=m"(*d)
1201                 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1202                 :"memory");
1203                 d += 24;
1204                 s += 8;
1205         }
1206         __asm __volatile(SFENCE:::"memory");
1207         __asm __volatile(EMMS:::"memory");
1208 #endif
1209         while(s < end)
1210         {
1211                 register uint16_t bgr;
1212                 bgr = *s++;
1213                 *d++ = (bgr&0x1F)<<3;
1214                 *d++ = (bgr&0x7E0)>>3;
1215                 *d++ = (bgr&0xF800)>>8;
1216         }
1217 }
1218
1219 static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_size)
1220 {
1221         const uint16_t *end;
1222 #ifdef HAVE_MMX
1223         const uint16_t *mm_end;
1224 #endif
1225         uint8_t *d = (uint8_t *)dst;
1226         const uint16_t *s = (const uint16_t *)src;
1227         end = s + src_size/2;
1228 #ifdef HAVE_MMX
1229         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1230         __asm __volatile("pxor  %%mm7,%%mm7\n\t":::"memory");
1231         mm_end = end - 3;
1232         while(s < mm_end)
1233         {
1234             __asm __volatile(
1235                 PREFETCH" 32%1\n\t"
1236                 "movq   %1, %%mm0\n\t"
1237                 "movq   %1, %%mm1\n\t"
1238                 "movq   %1, %%mm2\n\t"
1239                 "pand   %2, %%mm0\n\t"
1240                 "pand   %3, %%mm1\n\t"
1241                 "pand   %4, %%mm2\n\t"
1242                 "psllq  $3, %%mm0\n\t"
1243                 "psrlq  $2, %%mm1\n\t"
1244                 "psrlq  $7, %%mm2\n\t"
1245                 "movq   %%mm0, %%mm3\n\t"
1246                 "movq   %%mm1, %%mm4\n\t"
1247                 "movq   %%mm2, %%mm5\n\t"
1248                 "punpcklwd %%mm7, %%mm0\n\t"
1249                 "punpcklwd %%mm7, %%mm1\n\t"
1250                 "punpcklwd %%mm7, %%mm2\n\t"
1251                 "punpckhwd %%mm7, %%mm3\n\t"
1252                 "punpckhwd %%mm7, %%mm4\n\t"
1253                 "punpckhwd %%mm7, %%mm5\n\t"
1254                 "psllq  $8, %%mm1\n\t"
1255                 "psllq  $16, %%mm2\n\t"
1256                 "por    %%mm1, %%mm0\n\t"
1257                 "por    %%mm2, %%mm0\n\t"
1258                 "psllq  $8, %%mm4\n\t"
1259                 "psllq  $16, %%mm5\n\t"
1260                 "por    %%mm4, %%mm3\n\t"
1261                 "por    %%mm5, %%mm3\n\t"
1262                 MOVNTQ" %%mm0, %0\n\t"
1263                 MOVNTQ" %%mm3, 8%0\n\t"
1264                 :"=m"(*d)
1265                 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r)
1266                 :"memory");
1267                 d += 16;
1268                 s += 4;
1269         }
1270         __asm __volatile(SFENCE:::"memory");
1271         __asm __volatile(EMMS:::"memory");
1272 #endif
1273         while(s < end)
1274         {
1275 #if 0 //slightly slower on athlon
1276                 int bgr= *s++;
1277                 *((uint32_t*)d)++ = ((bgr&0x1F)<<3) + ((bgr&0x3E0)<<6) + ((bgr&0x7C00)<<9);
1278 #else
1279                 register uint16_t bgr;
1280                 bgr = *s++;
1281 #ifdef WORDS_BIGENDIAN
1282                 *d++ = 0;
1283                 *d++ = (bgr&0x7C00)>>7;
1284                 *d++ = (bgr&0x3E0)>>2;
1285                 *d++ = (bgr&0x1F)<<3;
1286 #else
1287                 *d++ = (bgr&0x1F)<<3;
1288                 *d++ = (bgr&0x3E0)>>2;
1289                 *d++ = (bgr&0x7C00)>>7;
1290                 *d++ = 0;
1291 #endif
1292
1293 #endif
1294         }
1295 }
1296
1297 static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_size)
1298 {
1299         const uint16_t *end;
1300 #ifdef HAVE_MMX
1301         const uint16_t *mm_end;
1302 #endif
1303         uint8_t *d = (uint8_t *)dst;
1304         const uint16_t *s = (uint16_t *)src;
1305         end = s + src_size/2;
1306 #ifdef HAVE_MMX
1307         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1308         __asm __volatile("pxor  %%mm7,%%mm7\n\t":::"memory");
1309         mm_end = end - 3;
1310         while(s < mm_end)
1311         {
1312             __asm __volatile(
1313                 PREFETCH" 32%1\n\t"
1314                 "movq   %1, %%mm0\n\t"
1315                 "movq   %1, %%mm1\n\t"
1316                 "movq   %1, %%mm2\n\t"
1317                 "pand   %2, %%mm0\n\t"
1318                 "pand   %3, %%mm1\n\t"
1319                 "pand   %4, %%mm2\n\t"
1320                 "psllq  $3, %%mm0\n\t"
1321                 "psrlq  $3, %%mm1\n\t"
1322                 "psrlq  $8, %%mm2\n\t"
1323                 "movq   %%mm0, %%mm3\n\t"
1324                 "movq   %%mm1, %%mm4\n\t"
1325                 "movq   %%mm2, %%mm5\n\t"
1326                 "punpcklwd %%mm7, %%mm0\n\t"
1327                 "punpcklwd %%mm7, %%mm1\n\t"
1328                 "punpcklwd %%mm7, %%mm2\n\t"
1329                 "punpckhwd %%mm7, %%mm3\n\t"
1330                 "punpckhwd %%mm7, %%mm4\n\t"
1331                 "punpckhwd %%mm7, %%mm5\n\t"
1332                 "psllq  $8, %%mm1\n\t"
1333                 "psllq  $16, %%mm2\n\t"
1334                 "por    %%mm1, %%mm0\n\t"
1335                 "por    %%mm2, %%mm0\n\t"
1336                 "psllq  $8, %%mm4\n\t"
1337                 "psllq  $16, %%mm5\n\t"
1338                 "por    %%mm4, %%mm3\n\t"
1339                 "por    %%mm5, %%mm3\n\t"
1340                 MOVNTQ" %%mm0, %0\n\t"
1341                 MOVNTQ" %%mm3, 8%0\n\t"
1342                 :"=m"(*d)
1343                 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r)
1344                 :"memory");
1345                 d += 16;
1346                 s += 4;
1347         }
1348         __asm __volatile(SFENCE:::"memory");
1349         __asm __volatile(EMMS:::"memory");
1350 #endif
1351         while(s < end)
1352         {
1353                 register uint16_t bgr;
1354                 bgr = *s++;
1355 #ifdef WORDS_BIGENDIAN
1356                 *d++ = 0;
1357                 *d++ = (bgr&0xF800)>>8;
1358                 *d++ = (bgr&0x7E0)>>3;
1359                 *d++ = (bgr&0x1F)<<3;
1360 #else
1361                 *d++ = (bgr&0x1F)<<3;
1362                 *d++ = (bgr&0x7E0)>>3;
1363                 *d++ = (bgr&0xF800)>>8;
1364                 *d++ = 0;
1365 #endif
1366         }
1367 }
1368
1369 static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, long src_size)
1370 {
1371         long idx = 15 - src_size;
1372         uint8_t *s = (uint8_t *) src-idx, *d = dst-idx;
1373 #ifdef HAVE_MMX
1374         __asm __volatile(
1375                 "       test %0, %0                     \n"
1376                 "       jns 2f                          \n"
1377                 "       "PREFETCH" (%1, %0)             \n"
1378                 "       movq %3, %%mm7                  \n"
1379                 "       pxor %4, %%mm7                  \n"
1380                 "       movq %%mm7, %%mm6               \n"
1381                 "       pxor %5, %%mm7                  \n"
1382                         ASMALIGN(4)
1383                 "1:                                     \n"
1384                 "       "PREFETCH" 32(%1, %0)           \n"
1385                 "       movq (%1, %0), %%mm0            \n"
1386                 "       movq 8(%1, %0), %%mm1           \n"
1387 # ifdef HAVE_MMX2
1388                 "       pshufw $177, %%mm0, %%mm3       \n"
1389                 "       pshufw $177, %%mm1, %%mm5       \n"
1390                 "       pand %%mm7, %%mm0               \n"
1391                 "       pand %%mm6, %%mm3               \n"
1392                 "       pand %%mm7, %%mm1               \n"
1393                 "       pand %%mm6, %%mm5               \n"
1394                 "       por %%mm3, %%mm0                \n"
1395                 "       por %%mm5, %%mm1                \n"
1396 # else
1397                 "       movq %%mm0, %%mm2               \n"
1398                 "       movq %%mm1, %%mm4               \n"
1399                 "       pand %%mm7, %%mm0               \n"
1400                 "       pand %%mm6, %%mm2               \n"
1401                 "       pand %%mm7, %%mm1               \n"
1402                 "       pand %%mm6, %%mm4               \n"
1403                 "       movq %%mm2, %%mm3               \n"
1404                 "       movq %%mm4, %%mm5               \n"
1405                 "       pslld $16, %%mm2                \n"
1406                 "       psrld $16, %%mm3                \n"
1407                 "       pslld $16, %%mm4                \n"
1408                 "       psrld $16, %%mm5                \n"
1409                 "       por %%mm2, %%mm0                \n"
1410                 "       por %%mm4, %%mm1                \n"
1411                 "       por %%mm3, %%mm0                \n"
1412                 "       por %%mm5, %%mm1                \n"
1413 # endif
1414                 "       "MOVNTQ" %%mm0, (%2, %0)        \n"
1415                 "       "MOVNTQ" %%mm1, 8(%2, %0)       \n"
1416                 "       add $16, %0                     \n"
1417                 "       js 1b                           \n"
1418                 "       "SFENCE"                        \n"
1419                 "       "EMMS"                          \n"
1420                 "2:                                     \n"
1421                 : "+&r"(idx)
1422                 : "r" (s), "r" (d), "m" (mask32b), "m" (mask32r), "m" (mmx_one)
1423                 : "memory");
1424 #endif
1425         for (; idx<15; idx+=4) {
1426                 register int v = *(uint32_t *)&s[idx], g = v & 0xff00;
1427                 v &= 0xff00ff;
1428                 *(uint32_t *)&d[idx] = (v>>16) + g + (v<<16);
1429         }
1430 }
1431
1432 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, long src_size)
1433 {
1434         unsigned i;
1435 #ifdef HAVE_MMX
1436         long mmx_size= 23 - src_size;
1437         asm volatile (
1438                 "movq "MANGLE(mask24r)", %%mm5  \n\t"
1439                 "movq "MANGLE(mask24g)", %%mm6  \n\t"
1440                 "movq "MANGLE(mask24b)", %%mm7  \n\t"
1441                 ASMALIGN(4)
1442                 "1:                             \n\t"
1443                 PREFETCH" 32(%1, %%"REG_a")     \n\t"
1444                 "movq   (%1, %%"REG_a"), %%mm0  \n\t" // BGR BGR BG
1445                 "movq   (%1, %%"REG_a"), %%mm1  \n\t" // BGR BGR BG
1446                 "movq  2(%1, %%"REG_a"), %%mm2  \n\t" // R BGR BGR B
1447                 "psllq $16, %%mm0               \n\t" // 00 BGR BGR
1448                 "pand %%mm5, %%mm0              \n\t"
1449                 "pand %%mm6, %%mm1              \n\t"
1450                 "pand %%mm7, %%mm2              \n\t"
1451                 "por %%mm0, %%mm1               \n\t"
1452                 "por %%mm2, %%mm1               \n\t"                
1453                 "movq  6(%1, %%"REG_a"), %%mm0  \n\t" // BGR BGR BG
1454                 MOVNTQ" %%mm1,   (%2, %%"REG_a")\n\t" // RGB RGB RG
1455                 "movq  8(%1, %%"REG_a"), %%mm1  \n\t" // R BGR BGR B
1456                 "movq 10(%1, %%"REG_a"), %%mm2  \n\t" // GR BGR BGR
1457                 "pand %%mm7, %%mm0              \n\t"
1458                 "pand %%mm5, %%mm1              \n\t"
1459                 "pand %%mm6, %%mm2              \n\t"
1460                 "por %%mm0, %%mm1               \n\t"
1461                 "por %%mm2, %%mm1               \n\t"                
1462                 "movq 14(%1, %%"REG_a"), %%mm0  \n\t" // R BGR BGR B
1463                 MOVNTQ" %%mm1,  8(%2, %%"REG_a")\n\t" // B RGB RGB R
1464                 "movq 16(%1, %%"REG_a"), %%mm1  \n\t" // GR BGR BGR
1465                 "movq 18(%1, %%"REG_a"), %%mm2  \n\t" // BGR BGR BG
1466                 "pand %%mm6, %%mm0              \n\t"
1467                 "pand %%mm7, %%mm1              \n\t"
1468                 "pand %%mm5, %%mm2              \n\t"
1469                 "por %%mm0, %%mm1               \n\t"
1470                 "por %%mm2, %%mm1               \n\t"                
1471                 MOVNTQ" %%mm1, 16(%2, %%"REG_a")\n\t"
1472                 "add $24, %%"REG_a"             \n\t"
1473                 " js 1b                         \n\t"
1474                 : "+a" (mmx_size)
1475                 : "r" (src-mmx_size), "r"(dst-mmx_size)
1476         );
1477
1478         __asm __volatile(SFENCE:::"memory");
1479         __asm __volatile(EMMS:::"memory");
1480
1481         if(mmx_size==23) return; //finihsed, was multiple of 8
1482
1483         src+= src_size;
1484         dst+= src_size;
1485         src_size= 23-mmx_size;
1486         src-= src_size;
1487         dst-= src_size;
1488 #endif
1489         for(i=0; i<src_size; i+=3)
1490         {
1491                 register uint8_t x;
1492                 x          = src[i + 2];
1493                 dst[i + 1] = src[i + 1];
1494                 dst[i + 2] = src[i + 0];
1495                 dst[i + 0] = x;
1496         }
1497 }
1498
1499 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1500         long width, long height,
1501         long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1502 {
1503         long y;
1504         const long chromWidth= width>>1;
1505         for(y=0; y<height; y++)
1506         {
1507 #ifdef HAVE_MMX
1508 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1509                 asm volatile(
1510                         "xor %%"REG_a", %%"REG_a"       \n\t"
1511                         ASMALIGN(4)
1512                         "1:                             \n\t"
1513                         PREFETCH" 32(%1, %%"REG_a", 2)  \n\t"
1514                         PREFETCH" 32(%2, %%"REG_a")     \n\t"
1515                         PREFETCH" 32(%3, %%"REG_a")     \n\t"
1516                         "movq (%2, %%"REG_a"), %%mm0    \n\t" // U(0)
1517                         "movq %%mm0, %%mm2              \n\t" // U(0)
1518                         "movq (%3, %%"REG_a"), %%mm1    \n\t" // V(0)
1519                         "punpcklbw %%mm1, %%mm0         \n\t" // UVUV UVUV(0)
1520                         "punpckhbw %%mm1, %%mm2         \n\t" // UVUV UVUV(8)
1521
1522                         "movq (%1, %%"REG_a",2), %%mm3  \n\t" // Y(0)
1523                         "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1524                         "movq %%mm3, %%mm4              \n\t" // Y(0)
1525                         "movq %%mm5, %%mm6              \n\t" // Y(8)
1526                         "punpcklbw %%mm0, %%mm3         \n\t" // YUYV YUYV(0)
1527                         "punpckhbw %%mm0, %%mm4         \n\t" // YUYV YUYV(4)
1528                         "punpcklbw %%mm2, %%mm5         \n\t" // YUYV YUYV(8)
1529                         "punpckhbw %%mm2, %%mm6         \n\t" // YUYV YUYV(12)
1530
1531                         MOVNTQ" %%mm3, (%0, %%"REG_a", 4)\n\t"
1532                         MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1533                         MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4)\n\t"
1534                         MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1535
1536                         "add $8, %%"REG_a"              \n\t"
1537                         "cmp %4, %%"REG_a"              \n\t"
1538                         " jb 1b                         \n\t"
1539                         ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1540                         : "%"REG_a
1541                 );
1542 #else
1543
1544 #if defined ARCH_ALPHA && defined HAVE_MVI
1545 #define pl2yuy2(n)                                      \
1546         y1 = yc[n];                                     \
1547         y2 = yc2[n];                                    \
1548         u = uc[n];                                      \
1549         v = vc[n];                                      \
1550         asm("unpkbw %1, %0" : "=r"(y1) : "r"(y1));      \
1551         asm("unpkbw %1, %0" : "=r"(y2) : "r"(y2));      \
1552         asm("unpkbl %1, %0" : "=r"(u) : "r"(u));        \
1553         asm("unpkbl %1, %0" : "=r"(v) : "r"(v));        \
1554         yuv1 = (u << 8) + (v << 24);                    \
1555         yuv2 = yuv1 + y2;                               \
1556         yuv1 += y1;                                     \
1557         qdst[n] = yuv1;                                 \
1558         qdst2[n] = yuv2;
1559
1560                 int i;
1561                 uint64_t *qdst = (uint64_t *) dst;
1562                 uint64_t *qdst2 = (uint64_t *) (dst + dstStride);
1563                 const uint32_t *yc = (uint32_t *) ysrc;
1564                 const uint32_t *yc2 = (uint32_t *) (ysrc + lumStride);
1565                 const uint16_t *uc = (uint16_t*) usrc, *vc = (uint16_t*) vsrc;
1566                 for(i = 0; i < chromWidth; i += 8){
1567                         uint64_t y1, y2, yuv1, yuv2;
1568                         uint64_t u, v;
1569                         /* Prefetch */
1570                         asm("ldq $31,64(%0)" :: "r"(yc));
1571                         asm("ldq $31,64(%0)" :: "r"(yc2));
1572                         asm("ldq $31,64(%0)" :: "r"(uc));
1573                         asm("ldq $31,64(%0)" :: "r"(vc));
1574
1575                         pl2yuy2(0);
1576                         pl2yuy2(1);
1577                         pl2yuy2(2);
1578                         pl2yuy2(3);
1579
1580                         yc += 4;
1581                         yc2 += 4;
1582                         uc += 4;
1583                         vc += 4;
1584                         qdst += 4;
1585                         qdst2 += 4;
1586                 }
1587                 y++;
1588                 ysrc += lumStride;
1589                 dst += dstStride;
1590
1591 #elif __WORDSIZE >= 64
1592                 int i;
1593                 uint64_t *ldst = (uint64_t *) dst;
1594                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1595                 for(i = 0; i < chromWidth; i += 2){
1596                         uint64_t k, l;
1597                         k = yc[0] + (uc[0] << 8) +
1598                             (yc[1] << 16) + (vc[0] << 24);
1599                         l = yc[2] + (uc[1] << 8) +
1600                             (yc[3] << 16) + (vc[1] << 24);
1601                         *ldst++ = k + (l << 32);
1602                         yc += 4;
1603                         uc += 2;
1604                         vc += 2;
1605                 }
1606
1607 #else
1608                 int i, *idst = (int32_t *) dst;
1609                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1610                 for(i = 0; i < chromWidth; i++){
1611 #ifdef WORDS_BIGENDIAN
1612                         *idst++ = (yc[0] << 24)+ (uc[0] << 16) +
1613                             (yc[1] << 8) + (vc[0] << 0);
1614 #else
1615                         *idst++ = yc[0] + (uc[0] << 8) +
1616                             (yc[1] << 16) + (vc[0] << 24);
1617 #endif
1618                         yc += 2;
1619                         uc++;
1620                         vc++;
1621                 }
1622 #endif
1623 #endif
1624                 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1625                 {
1626                         usrc += chromStride;
1627                         vsrc += chromStride;
1628                 }
1629                 ysrc += lumStride;
1630                 dst += dstStride;
1631         }
1632 #ifdef HAVE_MMX
1633 asm(    EMMS" \n\t"
1634         SFENCE" \n\t"
1635         :::"memory");
1636 #endif
1637 }
1638
1639 /**
1640  *
1641  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1642  * problem for anyone then tell me, and ill fix it)
1643  */
1644 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1645         long width, long height,
1646         long lumStride, long chromStride, long dstStride)
1647 {
1648         //FIXME interpolate chroma
1649         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1650 }
1651
1652 static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1653         long width, long height,
1654         long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1655 {
1656         long y;
1657         const long chromWidth= width>>1;
1658         for(y=0; y<height; y++)
1659         {
1660 #ifdef HAVE_MMX
1661 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1662                 asm volatile(
1663                         "xor %%"REG_a", %%"REG_a"       \n\t"
1664                         ASMALIGN(4)
1665                         "1:                             \n\t"
1666                         PREFETCH" 32(%1, %%"REG_a", 2)  \n\t"
1667                         PREFETCH" 32(%2, %%"REG_a")     \n\t"
1668                         PREFETCH" 32(%3, %%"REG_a")     \n\t"
1669                         "movq (%2, %%"REG_a"), %%mm0    \n\t" // U(0)
1670                         "movq %%mm0, %%mm2              \n\t" // U(0)
1671                         "movq (%3, %%"REG_a"), %%mm1    \n\t" // V(0)
1672                         "punpcklbw %%mm1, %%mm0         \n\t" // UVUV UVUV(0)
1673                         "punpckhbw %%mm1, %%mm2         \n\t" // UVUV UVUV(8)
1674
1675                         "movq (%1, %%"REG_a",2), %%mm3  \n\t" // Y(0)
1676                         "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1677                         "movq %%mm0, %%mm4              \n\t" // Y(0)
1678                         "movq %%mm2, %%mm6              \n\t" // Y(8)
1679                         "punpcklbw %%mm3, %%mm0         \n\t" // YUYV YUYV(0)
1680                         "punpckhbw %%mm3, %%mm4         \n\t" // YUYV YUYV(4)
1681                         "punpcklbw %%mm5, %%mm2         \n\t" // YUYV YUYV(8)
1682                         "punpckhbw %%mm5, %%mm6         \n\t" // YUYV YUYV(12)
1683
1684                         MOVNTQ" %%mm0, (%0, %%"REG_a", 4)\n\t"
1685                         MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1686                         MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4)\n\t"
1687                         MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1688
1689                         "add $8, %%"REG_a"              \n\t"
1690                         "cmp %4, %%"REG_a"              \n\t"
1691                         " jb 1b                         \n\t"
1692                         ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1693                         : "%"REG_a
1694                 );
1695 #else
1696 //FIXME adapt the alpha asm code from yv12->yuy2
1697
1698 #if __WORDSIZE >= 64
1699                 int i;
1700                 uint64_t *ldst = (uint64_t *) dst;
1701                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1702                 for(i = 0; i < chromWidth; i += 2){
1703                         uint64_t k, l;
1704                         k = uc[0] + (yc[0] << 8) +
1705                             (vc[0] << 16) + (yc[1] << 24);
1706                         l = uc[1] + (yc[2] << 8) +
1707                             (vc[1] << 16) + (yc[3] << 24);
1708                         *ldst++ = k + (l << 32);
1709                         yc += 4;
1710                         uc += 2;
1711                         vc += 2;
1712                 }
1713
1714 #else
1715                 int i, *idst = (int32_t *) dst;
1716                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1717                 for(i = 0; i < chromWidth; i++){
1718 #ifdef WORDS_BIGENDIAN
1719                         *idst++ = (uc[0] << 24)+ (yc[0] << 16) +
1720                             (vc[0] << 8) + (yc[1] << 0);
1721 #else
1722                         *idst++ = uc[0] + (yc[0] << 8) +
1723                             (vc[0] << 16) + (yc[1] << 24);
1724 #endif
1725                         yc += 2;
1726                         uc++;
1727                         vc++;
1728                 }
1729 #endif
1730 #endif
1731                 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1732                 {
1733                         usrc += chromStride;
1734                         vsrc += chromStride;
1735                 }
1736                 ysrc += lumStride;
1737                 dst += dstStride;
1738         }
1739 #ifdef HAVE_MMX
1740 asm(    EMMS" \n\t"
1741         SFENCE" \n\t"
1742         :::"memory");
1743 #endif
1744 }
1745
1746 /**
1747  *
1748  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1749  * problem for anyone then tell me, and ill fix it)
1750  */
1751 static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1752         long width, long height,
1753         long lumStride, long chromStride, long dstStride)
1754 {
1755         //FIXME interpolate chroma
1756         RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1757 }
1758
1759 /**
1760  *
1761  * width should be a multiple of 16
1762  */
1763 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1764         long width, long height,
1765         long lumStride, long chromStride, long dstStride)
1766 {
1767         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1768 }
1769
1770 /**
1771  *
1772  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1773  * problem for anyone then tell me, and ill fix it)
1774  */
1775 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1776         long width, long height,
1777         long lumStride, long chromStride, long srcStride)
1778 {
1779         long y;
1780         const long chromWidth= width>>1;
1781         for(y=0; y<height; y+=2)
1782         {
1783 #ifdef HAVE_MMX
1784                 asm volatile(
1785                         "xor %%"REG_a", %%"REG_a"       \n\t"
1786                         "pcmpeqw %%mm7, %%mm7           \n\t"
1787                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
1788                         ASMALIGN(4)
1789                         "1:                             \n\t"
1790                         PREFETCH" 64(%0, %%"REG_a", 4)  \n\t"
1791                         "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1792                         "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1793                         "movq %%mm0, %%mm2              \n\t" // YUYV YUYV(0)
1794                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(4)
1795                         "psrlw $8, %%mm0                \n\t" // U0V0 U0V0(0)
1796                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(4)
1797                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(0)
1798                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(4)
1799                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
1800                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
1801
1802                         MOVNTQ" %%mm2, (%1, %%"REG_a", 2)\n\t"
1803
1804                         "movq 16(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(8)
1805                         "movq 24(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(12)
1806                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(8)
1807                         "movq %%mm2, %%mm4              \n\t" // YUYV YUYV(12)
1808                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(8)
1809                         "psrlw $8, %%mm2                \n\t" // U0V0 U0V0(12)
1810                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(8)
1811                         "pand %%mm7, %%mm4              \n\t" // Y0Y0 Y0Y0(12)
1812                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
1813                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
1814
1815                         MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2)\n\t"
1816
1817                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
1818                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
1819                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
1820                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
1821                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
1822                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
1823                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
1824                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
1825
1826                         MOVNTQ" %%mm0, (%3, %%"REG_a")  \n\t"
1827                         MOVNTQ" %%mm2, (%2, %%"REG_a")  \n\t"
1828
1829                         "add $8, %%"REG_a"              \n\t"
1830                         "cmp %4, %%"REG_a"              \n\t"
1831                         " jb 1b                         \n\t"
1832                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1833                         : "memory", "%"REG_a
1834                 );
1835
1836                 ydst += lumStride;
1837                 src  += srcStride;
1838
1839                 asm volatile(
1840                         "xor %%"REG_a", %%"REG_a"       \n\t"
1841                         ASMALIGN(4)
1842                         "1:                             \n\t"
1843                         PREFETCH" 64(%0, %%"REG_a", 4)  \n\t"
1844                         "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1845                         "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1846                         "movq 16(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(8)
1847                         "movq 24(%0, %%"REG_a", 4), %%mm3\n\t" // YUYV YUYV(12)
1848                         "pand %%mm7, %%mm0              \n\t" // Y0Y0 Y0Y0(0)
1849                         "pand %%mm7, %%mm1              \n\t" // Y0Y0 Y0Y0(4)
1850                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(8)
1851                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(12)
1852                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
1853                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
1854
1855                         MOVNTQ" %%mm0, (%1, %%"REG_a", 2)\n\t"
1856                         MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2)\n\t"
1857
1858                         "add $8, %%"REG_a"              \n\t"
1859                         "cmp %4, %%"REG_a"              \n\t"
1860                         " jb 1b                         \n\t"
1861
1862                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1863                         : "memory", "%"REG_a
1864                 );
1865 #else
1866                 long i;
1867                 for(i=0; i<chromWidth; i++)
1868                 {
1869                         ydst[2*i+0]     = src[4*i+0];
1870                         udst[i]         = src[4*i+1];
1871                         ydst[2*i+1]     = src[4*i+2];
1872                         vdst[i]         = src[4*i+3];
1873                 }
1874                 ydst += lumStride;
1875                 src  += srcStride;
1876
1877                 for(i=0; i<chromWidth; i++)
1878                 {
1879                         ydst[2*i+0]     = src[4*i+0];
1880                         ydst[2*i+1]     = src[4*i+2];
1881                 }
1882 #endif
1883                 udst += chromStride;
1884                 vdst += chromStride;
1885                 ydst += lumStride;
1886                 src  += srcStride;
1887         }
1888 #ifdef HAVE_MMX
1889 asm volatile(   EMMS" \n\t"
1890                 SFENCE" \n\t"
1891                 :::"memory");
1892 #endif
1893 }
1894
1895 static inline void RENAME(yvu9toyv12)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,
1896         uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1897         long width, long height, long lumStride, long chromStride)
1898 {
1899         /* Y Plane */
1900         memcpy(ydst, ysrc, width*height);
1901
1902         /* XXX: implement upscaling for U,V */
1903 }
1904
1905 static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, long srcWidth, long srcHeight, long srcStride, long dstStride)
1906 {
1907         long x,y;
1908         
1909         dst[0]= src[0];
1910         
1911         // first line
1912         for(x=0; x<srcWidth-1; x++){
1913                 dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
1914                 dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
1915         }
1916         dst[2*srcWidth-1]= src[srcWidth-1];
1917         
1918         dst+= dstStride;
1919
1920         for(y=1; y<srcHeight; y++){
1921 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
1922                 const long mmxSize= srcWidth&~15;
1923                 asm volatile(
1924                         "mov %4, %%"REG_a"              \n\t"
1925                         "1:                             \n\t"
1926                         "movq (%0, %%"REG_a"), %%mm0    \n\t"
1927                         "movq (%1, %%"REG_a"), %%mm1    \n\t"
1928                         "movq 1(%0, %%"REG_a"), %%mm2   \n\t"
1929                         "movq 1(%1, %%"REG_a"), %%mm3   \n\t"
1930                         "movq -1(%0, %%"REG_a"), %%mm4  \n\t"
1931                         "movq -1(%1, %%"REG_a"), %%mm5  \n\t"
1932                         PAVGB" %%mm0, %%mm5             \n\t"
1933                         PAVGB" %%mm0, %%mm3             \n\t"
1934                         PAVGB" %%mm0, %%mm5             \n\t"
1935                         PAVGB" %%mm0, %%mm3             \n\t"
1936                         PAVGB" %%mm1, %%mm4             \n\t"
1937                         PAVGB" %%mm1, %%mm2             \n\t"
1938                         PAVGB" %%mm1, %%mm4             \n\t"
1939                         PAVGB" %%mm1, %%mm2             \n\t"
1940                         "movq %%mm5, %%mm7              \n\t"
1941                         "movq %%mm4, %%mm6              \n\t"
1942                         "punpcklbw %%mm3, %%mm5         \n\t"
1943                         "punpckhbw %%mm3, %%mm7         \n\t"
1944                         "punpcklbw %%mm2, %%mm4         \n\t"
1945                         "punpckhbw %%mm2, %%mm6         \n\t"
1946 #if 1
1947                         MOVNTQ" %%mm5, (%2, %%"REG_a", 2)\n\t"
1948                         MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1949                         MOVNTQ" %%mm4, (%3, %%"REG_a", 2)\n\t"
1950                         MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1951 #else
1952                         "movq %%mm5, (%2, %%"REG_a", 2) \n\t"
1953                         "movq %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1954                         "movq %%mm4, (%3, %%"REG_a", 2) \n\t"
1955                         "movq %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1956 #endif
1957                         "add $8, %%"REG_a"              \n\t"
1958                         " js 1b                         \n\t"
1959                         :: "r" (src + mmxSize  ), "r" (src + srcStride + mmxSize  ),
1960                            "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
1961                            "g" (-mmxSize)
1962                         : "%"REG_a
1963
1964                 );
1965 #else
1966                 const long mmxSize=1;
1967 #endif
1968                 dst[0        ]= (3*src[0] +   src[srcStride])>>2;
1969                 dst[dstStride]= (  src[0] + 3*src[srcStride])>>2;
1970
1971                 for(x=mmxSize-1; x<srcWidth-1; x++){
1972                         dst[2*x          +1]= (3*src[x+0] +   src[x+srcStride+1])>>2;
1973                         dst[2*x+dstStride+2]= (  src[x+0] + 3*src[x+srcStride+1])>>2;
1974                         dst[2*x+dstStride+1]= (  src[x+1] + 3*src[x+srcStride  ])>>2;
1975                         dst[2*x          +2]= (3*src[x+1] +   src[x+srcStride  ])>>2;
1976                 }
1977                 dst[srcWidth*2 -1            ]= (3*src[srcWidth-1] +   src[srcWidth-1 + srcStride])>>2;
1978                 dst[srcWidth*2 -1 + dstStride]= (  src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
1979
1980                 dst+=dstStride*2;
1981                 src+=srcStride;
1982         }
1983         
1984         // last line
1985 #if 1
1986         dst[0]= src[0];
1987         
1988         for(x=0; x<srcWidth-1; x++){
1989                 dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
1990                 dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
1991         }
1992         dst[2*srcWidth-1]= src[srcWidth-1];
1993 #else
1994         for(x=0; x<srcWidth; x++){
1995                 dst[2*x+0]=
1996                 dst[2*x+1]= src[x];
1997         }
1998 #endif
1999
2000 #ifdef HAVE_MMX
2001 asm volatile(   EMMS" \n\t"
2002                 SFENCE" \n\t"
2003                 :::"memory");
2004 #endif
2005 }
2006
2007 /**
2008  *
2009  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
2010  * problem for anyone then tell me, and ill fix it)
2011  * chrominance data is only taken from every secound line others are ignored FIXME write HQ version
2012  */
2013 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2014         long width, long height,
2015         long lumStride, long chromStride, long srcStride)
2016 {
2017         long y;
2018         const long chromWidth= width>>1;
2019         for(y=0; y<height; y+=2)
2020         {
2021 #ifdef HAVE_MMX
2022                 asm volatile(
2023                         "xorl %%eax, %%eax              \n\t"
2024                         "pcmpeqw %%mm7, %%mm7           \n\t"
2025                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
2026                         ASMALIGN(4)
2027                         "1:                             \n\t"
2028                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
2029                         "movq (%0, %%eax, 4), %%mm0     \n\t" // UYVY UYVY(0)
2030                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // UYVY UYVY(4)
2031                         "movq %%mm0, %%mm2              \n\t" // UYVY UYVY(0)
2032                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(4)
2033                         "pand %%mm7, %%mm0              \n\t" // U0V0 U0V0(0)
2034                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(4)
2035                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(0)
2036                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(4)
2037                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
2038                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
2039
2040                         MOVNTQ" %%mm2, (%1, %%eax, 2)   \n\t"
2041
2042                         "movq 16(%0, %%eax, 4), %%mm1   \n\t" // UYVY UYVY(8)
2043                         "movq 24(%0, %%eax, 4), %%mm2   \n\t" // UYVY UYVY(12)
2044                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(8)
2045                         "movq %%mm2, %%mm4              \n\t" // UYVY UYVY(12)
2046                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(8)
2047                         "pand %%mm7, %%mm2              \n\t" // U0V0 U0V0(12)
2048                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(8)
2049                         "psrlw $8, %%mm4                \n\t" // Y0Y0 Y0Y0(12)
2050                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
2051                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
2052
2053                         MOVNTQ" %%mm3, 8(%1, %%eax, 2)  \n\t"
2054
2055                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
2056                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
2057                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
2058                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
2059                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
2060                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
2061                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
2062                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
2063
2064                         MOVNTQ" %%mm0, (%3, %%eax)      \n\t"
2065                         MOVNTQ" %%mm2, (%2, %%eax)      \n\t"
2066
2067                         "addl $8, %%eax                 \n\t"
2068                         "cmpl %4, %%eax                 \n\t"
2069                         " jb 1b                         \n\t"
2070                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2071                         : "memory", "%eax"
2072                 );
2073
2074                 ydst += lumStride;
2075                 src  += srcStride;
2076
2077                 asm volatile(
2078                         "xorl %%eax, %%eax              \n\t"
2079                         ASMALIGN(4)
2080                         "1:                             \n\t"
2081                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
2082                         "movq (%0, %%eax, 4), %%mm0     \n\t" // YUYV YUYV(0)
2083                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // YUYV YUYV(4)
2084                         "movq 16(%0, %%eax, 4), %%mm2   \n\t" // YUYV YUYV(8)
2085                         "movq 24(%0, %%eax, 4), %%mm3   \n\t" // YUYV YUYV(12)
2086                         "psrlw $8, %%mm0                \n\t" // Y0Y0 Y0Y0(0)
2087                         "psrlw $8, %%mm1                \n\t" // Y0Y0 Y0Y0(4)
2088                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(8)
2089                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(12)
2090                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
2091                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
2092
2093                         MOVNTQ" %%mm0, (%1, %%eax, 2)   \n\t"
2094                         MOVNTQ" %%mm2, 8(%1, %%eax, 2)  \n\t"
2095
2096                         "addl $8, %%eax                 \n\t"
2097                         "cmpl %4, %%eax                 \n\t"
2098                         " jb 1b                         \n\t"
2099
2100                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2101                         : "memory", "%eax"
2102                 );
2103 #else
2104                 long i;
2105                 for(i=0; i<chromWidth; i++)
2106                 {
2107                         udst[i]         = src[4*i+0];
2108                         ydst[2*i+0]     = src[4*i+1];
2109                         vdst[i]         = src[4*i+2];
2110                         ydst[2*i+1]     = src[4*i+3];
2111                 }
2112                 ydst += lumStride;
2113                 src  += srcStride;
2114
2115                 for(i=0; i<chromWidth; i++)
2116                 {
2117                         ydst[2*i+0]     = src[4*i+1];
2118                         ydst[2*i+1]     = src[4*i+3];
2119                 }
2120 #endif
2121                 udst += chromStride;
2122                 vdst += chromStride;
2123                 ydst += lumStride;
2124                 src  += srcStride;
2125         }
2126 #ifdef HAVE_MMX
2127 asm volatile(   EMMS" \n\t"
2128                 SFENCE" \n\t"
2129                 :::"memory");
2130 #endif
2131 }
2132
2133 /**
2134  *
2135  * height should be a multiple of 2 and width should be a multiple of 2 (if this is a
2136  * problem for anyone then tell me, and ill fix it)
2137  * chrominance data is only taken from every secound line others are ignored in the C version FIXME write HQ version
2138  */
2139 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2140         long width, long height,
2141         long lumStride, long chromStride, long srcStride)
2142 {
2143         long y;
2144         const long chromWidth= width>>1;
2145 #ifdef HAVE_MMX
2146         for(y=0; y<height-2; y+=2)
2147         {
2148                 long i;
2149                 for(i=0; i<2; i++)
2150                 {
2151                         asm volatile(
2152                                 "mov %2, %%"REG_a"              \n\t"
2153                                 "movq "MANGLE(bgr2YCoeff)", %%mm6               \n\t"
2154                                 "movq "MANGLE(w1111)", %%mm5            \n\t"
2155                                 "pxor %%mm7, %%mm7              \n\t"
2156                                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2157                                 ASMALIGN(4)
2158                                 "1:                             \n\t"
2159                                 PREFETCH" 64(%0, %%"REG_d")     \n\t"
2160                                 "movd (%0, %%"REG_d"), %%mm0    \n\t"
2161                                 "movd 3(%0, %%"REG_d"), %%mm1   \n\t"
2162                                 "punpcklbw %%mm7, %%mm0         \n\t"
2163                                 "punpcklbw %%mm7, %%mm1         \n\t"
2164                                 "movd 6(%0, %%"REG_d"), %%mm2   \n\t"
2165                                 "movd 9(%0, %%"REG_d"), %%mm3   \n\t"
2166                                 "punpcklbw %%mm7, %%mm2         \n\t"
2167                                 "punpcklbw %%mm7, %%mm3         \n\t"
2168                                 "pmaddwd %%mm6, %%mm0           \n\t"
2169                                 "pmaddwd %%mm6, %%mm1           \n\t"
2170                                 "pmaddwd %%mm6, %%mm2           \n\t"
2171                                 "pmaddwd %%mm6, %%mm3           \n\t"
2172 #ifndef FAST_BGR2YV12
2173                                 "psrad $8, %%mm0                \n\t"
2174                                 "psrad $8, %%mm1                \n\t"
2175                                 "psrad $8, %%mm2                \n\t"
2176                                 "psrad $8, %%mm3                \n\t"
2177 #endif
2178                                 "packssdw %%mm1, %%mm0          \n\t"
2179                                 "packssdw %%mm3, %%mm2          \n\t"
2180                                 "pmaddwd %%mm5, %%mm0           \n\t"
2181                                 "pmaddwd %%mm5, %%mm2           \n\t"
2182                                 "packssdw %%mm2, %%mm0          \n\t"
2183                                 "psraw $7, %%mm0                \n\t"
2184
2185                                 "movd 12(%0, %%"REG_d"), %%mm4  \n\t"
2186                                 "movd 15(%0, %%"REG_d"), %%mm1  \n\t"
2187                                 "punpcklbw %%mm7, %%mm4         \n\t"
2188                                 "punpcklbw %%mm7, %%mm1         \n\t"
2189                                 "movd 18(%0, %%"REG_d"), %%mm2  \n\t"
2190                                 "movd 21(%0, %%"REG_d"), %%mm3  \n\t"
2191                                 "punpcklbw %%mm7, %%mm2         \n\t"
2192                                 "punpcklbw %%mm7, %%mm3         \n\t"
2193                                 "pmaddwd %%mm6, %%mm4           \n\t"
2194                                 "pmaddwd %%mm6, %%mm1           \n\t"
2195                                 "pmaddwd %%mm6, %%mm2           \n\t"
2196                                 "pmaddwd %%mm6, %%mm3           \n\t"
2197 #ifndef FAST_BGR2YV12
2198                                 "psrad $8, %%mm4                \n\t"
2199                                 "psrad $8, %%mm1                \n\t"
2200                                 "psrad $8, %%mm2                \n\t"
2201                                 "psrad $8, %%mm3                \n\t"
2202 #endif
2203                                 "packssdw %%mm1, %%mm4          \n\t"
2204                                 "packssdw %%mm3, %%mm2          \n\t"
2205                                 "pmaddwd %%mm5, %%mm4           \n\t"
2206                                 "pmaddwd %%mm5, %%mm2           \n\t"
2207                                 "add $24, %%"REG_d"             \n\t"
2208                                 "packssdw %%mm2, %%mm4          \n\t"
2209                                 "psraw $7, %%mm4                \n\t"
2210
2211                                 "packuswb %%mm4, %%mm0          \n\t"
2212                                 "paddusb "MANGLE(bgr2YOffset)", %%mm0   \n\t"
2213
2214                                 MOVNTQ" %%mm0, (%1, %%"REG_a")  \n\t"
2215                                 "add $8, %%"REG_a"              \n\t"
2216                                 " js 1b                         \n\t"
2217                                 : : "r" (src+width*3), "r" (ydst+width), "g" (-width)
2218                                 : "%"REG_a, "%"REG_d
2219                         );
2220                         ydst += lumStride;
2221                         src  += srcStride;
2222                 }
2223                 src -= srcStride*2;
2224                 asm volatile(
2225                         "mov %4, %%"REG_a"              \n\t"
2226                         "movq "MANGLE(w1111)", %%mm5            \n\t"
2227                         "movq "MANGLE(bgr2UCoeff)", %%mm6               \n\t"
2228                         "pxor %%mm7, %%mm7              \n\t"
2229                         "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2230                         "add %%"REG_d", %%"REG_d"       \n\t"
2231                         ASMALIGN(4)
2232                         "1:                             \n\t"
2233                         PREFETCH" 64(%0, %%"REG_d")     \n\t"
2234                         PREFETCH" 64(%1, %%"REG_d")     \n\t"
2235 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2236                         "movq (%0, %%"REG_d"), %%mm0    \n\t"
2237                         "movq (%1, %%"REG_d"), %%mm1    \n\t"
2238                         "movq 6(%0, %%"REG_d"), %%mm2   \n\t"
2239                         "movq 6(%1, %%"REG_d"), %%mm3   \n\t"
2240                         PAVGB" %%mm1, %%mm0             \n\t"
2241                         PAVGB" %%mm3, %%mm2             \n\t"
2242                         "movq %%mm0, %%mm1              \n\t"
2243                         "movq %%mm2, %%mm3              \n\t"
2244                         "psrlq $24, %%mm0               \n\t"
2245                         "psrlq $24, %%mm2               \n\t"
2246                         PAVGB" %%mm1, %%mm0             \n\t"
2247                         PAVGB" %%mm3, %%mm2             \n\t"
2248                         "punpcklbw %%mm7, %%mm0         \n\t"
2249                         "punpcklbw %%mm7, %%mm2         \n\t"
2250 #else
2251                         "movd (%0, %%"REG_d"), %%mm0    \n\t"
2252                         "movd (%1, %%"REG_d"), %%mm1    \n\t"
2253                         "movd 3(%0, %%"REG_d"), %%mm2   \n\t"
2254                         "movd 3(%1, %%"REG_d"), %%mm3   \n\t"
2255                         "punpcklbw %%mm7, %%mm0         \n\t"
2256                         "punpcklbw %%mm7, %%mm1         \n\t"
2257                         "punpcklbw %%mm7, %%mm2         \n\t"
2258                         "punpcklbw %%mm7, %%mm3         \n\t"
2259                         "paddw %%mm1, %%mm0             \n\t"
2260                         "paddw %%mm3, %%mm2             \n\t"
2261                         "paddw %%mm2, %%mm0             \n\t"
2262                         "movd 6(%0, %%"REG_d"), %%mm4   \n\t"
2263                         "movd 6(%1, %%"REG_d"), %%mm1   \n\t"
2264                         "movd 9(%0, %%"REG_d"), %%mm2   \n\t"
2265                         "movd 9(%1, %%"REG_d"), %%mm3   \n\t"
2266                         "punpcklbw %%mm7, %%mm4         \n\t"
2267                         "punpcklbw %%mm7, %%mm1         \n\t"
2268                         "punpcklbw %%mm7, %%mm2         \n\t"
2269                         "punpcklbw %%mm7, %%mm3         \n\t"
2270                         "paddw %%mm1, %%mm4             \n\t"
2271                         "paddw %%mm3, %%mm2             \n\t"
2272                         "paddw %%mm4, %%mm2             \n\t"
2273                         "psrlw $2, %%mm0                \n\t"
2274                         "psrlw $2, %%mm2                \n\t"
2275 #endif
2276                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
2277                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
2278
2279                         "pmaddwd %%mm0, %%mm1           \n\t"
2280                         "pmaddwd %%mm2, %%mm3           \n\t"
2281                         "pmaddwd %%mm6, %%mm0           \n\t"
2282                         "pmaddwd %%mm6, %%mm2           \n\t"
2283 #ifndef FAST_BGR2YV12
2284                         "psrad $8, %%mm0                \n\t"
2285                         "psrad $8, %%mm1                \n\t"
2286                         "psrad $8, %%mm2                \n\t"
2287                         "psrad $8, %%mm3                \n\t"
2288 #endif
2289                         "packssdw %%mm2, %%mm0          \n\t"
2290                         "packssdw %%mm3, %%mm1          \n\t"
2291                         "pmaddwd %%mm5, %%mm0           \n\t"
2292                         "pmaddwd %%mm5, %%mm1           \n\t"
2293                         "packssdw %%mm1, %%mm0          \n\t" // V1 V0 U1 U0
2294                         "psraw $7, %%mm0                \n\t"
2295
2296 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2297                         "movq 12(%0, %%"REG_d"), %%mm4  \n\t"
2298                         "movq 12(%1, %%"REG_d"), %%mm1  \n\t"
2299                         "movq 18(%0, %%"REG_d"), %%mm2  \n\t"
2300                         "movq 18(%1, %%"REG_d"), %%mm3  \n\t"
2301                         PAVGB" %%mm1, %%mm4             \n\t"
2302                         PAVGB" %%mm3, %%mm2             \n\t"
2303                         "movq %%mm4, %%mm1              \n\t"
2304                         "movq %%mm2, %%mm3              \n\t"
2305                         "psrlq $24, %%mm4               \n\t"
2306                         "psrlq $24, %%mm2               \n\t"
2307                         PAVGB" %%mm1, %%mm4             \n\t"
2308                         PAVGB" %%mm3, %%mm2             \n\t"
2309                         "punpcklbw %%mm7, %%mm4         \n\t"
2310                         "punpcklbw %%mm7, %%mm2         \n\t"
2311 #else
2312                         "movd 12(%0, %%"REG_d"), %%mm4  \n\t"
2313                         "movd 12(%1, %%"REG_d"), %%mm1  \n\t"
2314                         "movd 15(%0, %%"REG_d"), %%mm2  \n\t"
2315                         "movd 15(%1, %%"REG_d"), %%mm3  \n\t"
2316                         "punpcklbw %%mm7, %%mm4         \n\t"
2317                         "punpcklbw %%mm7, %%mm1         \n\t"
2318                         "punpcklbw %%mm7, %%mm2         \n\t"
2319                         "punpcklbw %%mm7, %%mm3         \n\t"
2320                         "paddw %%mm1, %%mm4             \n\t"
2321                         "paddw %%mm3, %%mm2             \n\t"
2322                         "paddw %%mm2, %%mm4             \n\t"
2323                         "movd 18(%0, %%"REG_d"), %%mm5  \n\t"
2324                         "movd 18(%1, %%"REG_d"), %%mm1  \n\t"
2325                         "movd 21(%0, %%"REG_d"), %%mm2  \n\t"
2326                         "movd 21(%1, %%"REG_d"), %%mm3  \n\t"
2327                         "punpcklbw %%mm7, %%mm5         \n\t"
2328                         "punpcklbw %%mm7, %%mm1         \n\t"
2329                         "punpcklbw %%mm7, %%mm2         \n\t"
2330                         "punpcklbw %%mm7, %%mm3         \n\t"
2331                         "paddw %%mm1, %%mm5             \n\t"
2332                         "paddw %%mm3, %%mm2             \n\t"
2333                         "paddw %%mm5, %%mm2             \n\t"
2334                         "movq "MANGLE(w1111)", %%mm5            \n\t"
2335                         "psrlw $2, %%mm4                \n\t"
2336                         "psrlw $2, %%mm2                \n\t"
2337 #endif
2338                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
2339                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
2340
2341                         "pmaddwd %%mm4, %%mm1           \n\t"
2342                         "pmaddwd %%mm2, %%mm3           \n\t"
2343                         "pmaddwd %%mm6, %%mm4           \n\t"
2344                         "pmaddwd %%mm6, %%mm2           \n\t"
2345 #ifndef FAST_BGR2YV12
2346                         "psrad $8, %%mm4                \n\t"
2347                         "psrad $8, %%mm1                \n\t"
2348                         "psrad $8, %%mm2                \n\t"
2349                         "psrad $8, %%mm3                \n\t"
2350 #endif
2351                         "packssdw %%mm2, %%mm4          \n\t"
2352                         "packssdw %%mm3, %%mm1          \n\t"
2353                         "pmaddwd %%mm5, %%mm4           \n\t"
2354                         "pmaddwd %%mm5, %%mm1           \n\t"
2355                         "add $24, %%"REG_d"             \n\t"
2356                         "packssdw %%mm1, %%mm4          \n\t" // V3 V2 U3 U2
2357                         "psraw $7, %%mm4                \n\t"
2358
2359                         "movq %%mm0, %%mm1              \n\t"
2360                         "punpckldq %%mm4, %%mm0         \n\t"
2361                         "punpckhdq %%mm4, %%mm1         \n\t"
2362                         "packsswb %%mm1, %%mm0          \n\t"
2363                         "paddb "MANGLE(bgr2UVOffset)", %%mm0    \n\t"
2364                         "movd %%mm0, (%2, %%"REG_a")    \n\t"
2365                         "punpckhdq %%mm0, %%mm0         \n\t"
2366                         "movd %%mm0, (%3, %%"REG_a")    \n\t"
2367                         "add $4, %%"REG_a"              \n\t"
2368                         " js 1b                         \n\t"
2369                         : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth)
2370                         : "%"REG_a, "%"REG_d
2371                 );
2372
2373                 udst += chromStride;
2374                 vdst += chromStride;
2375                 src  += srcStride*2;
2376         }
2377
2378         asm volatile(   EMMS" \n\t"
2379                         SFENCE" \n\t"
2380                         :::"memory");
2381 #else
2382         y=0;
2383 #endif
2384         for(; y<height; y+=2)
2385         {
2386                 long i;
2387                 for(i=0; i<chromWidth; i++)
2388                 {
2389                         unsigned int b= src[6*i+0];
2390                         unsigned int g= src[6*i+1];
2391                         unsigned int r= src[6*i+2];
2392
2393                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2394                         unsigned int V  =  ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128;
2395                         unsigned int U  =  ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128;
2396
2397                         udst[i]         = U;
2398                         vdst[i]         = V;
2399                         ydst[2*i]       = Y;
2400
2401                         b= src[6*i+3];
2402                         g= src[6*i+4];
2403                         r= src[6*i+5];
2404
2405                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2406                         ydst[2*i+1]     = Y;
2407                 }
2408                 ydst += lumStride;
2409                 src  += srcStride;
2410
2411                 for(i=0; i<chromWidth; i++)
2412                 {
2413                         unsigned int b= src[6*i+0];
2414                         unsigned int g= src[6*i+1];
2415                         unsigned int r= src[6*i+2];
2416
2417                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2418
2419                         ydst[2*i]       = Y;
2420
2421                         b= src[6*i+3];
2422                         g= src[6*i+4];
2423                         r= src[6*i+5];
2424
2425                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2426                         ydst[2*i+1]     = Y;
2427                 }
2428                 udst += chromStride;
2429                 vdst += chromStride;
2430                 ydst += lumStride;
2431                 src  += srcStride;
2432         }
2433 }
2434
2435 void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest,
2436                             long width, long height, long src1Stride,
2437                             long src2Stride, long dstStride){
2438         long h;
2439
2440         for(h=0; h < height; h++)
2441         {
2442                 long w;
2443
2444 #ifdef HAVE_MMX
2445 #ifdef HAVE_SSE2
2446                 asm(
2447                         "xor %%"REG_a", %%"REG_a"       \n\t"
2448                         "1:                             \n\t"
2449                         PREFETCH" 64(%1, %%"REG_a")     \n\t"
2450                         PREFETCH" 64(%2, %%"REG_a")     \n\t"
2451                         "movdqa (%1, %%"REG_a"), %%xmm0 \n\t"
2452                         "movdqa (%1, %%"REG_a"), %%xmm1 \n\t"
2453                         "movdqa (%2, %%"REG_a"), %%xmm2 \n\t"
2454                         "punpcklbw %%xmm2, %%xmm0       \n\t"
2455                         "punpckhbw %%xmm2, %%xmm1       \n\t"
2456                         "movntdq %%xmm0, (%0, %%"REG_a", 2)\n\t"
2457                         "movntdq %%xmm1, 16(%0, %%"REG_a", 2)\n\t"
2458                         "add $16, %%"REG_a"             \n\t"
2459                         "cmp %3, %%"REG_a"              \n\t"
2460                         " jb 1b                         \n\t"
2461                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2462                         : "memory", "%"REG_a""
2463                 );
2464 #else
2465                 asm(
2466                         "xor %%"REG_a", %%"REG_a"       \n\t"
2467                         "1:                             \n\t"
2468                         PREFETCH" 64(%1, %%"REG_a")     \n\t"
2469                         PREFETCH" 64(%2, %%"REG_a")     \n\t"
2470                         "movq (%1, %%"REG_a"), %%mm0    \n\t"
2471                         "movq 8(%1, %%"REG_a"), %%mm2   \n\t"
2472                         "movq %%mm0, %%mm1              \n\t"
2473                         "movq %%mm2, %%mm3              \n\t"
2474                         "movq (%2, %%"REG_a"), %%mm4    \n\t"
2475                         "movq 8(%2, %%"REG_a"), %%mm5   \n\t"
2476                         "punpcklbw %%mm4, %%mm0         \n\t"
2477                         "punpckhbw %%mm4, %%mm1         \n\t"
2478                         "punpcklbw %%mm5, %%mm2         \n\t"
2479                         "punpckhbw %%mm5, %%mm3         \n\t"
2480                         MOVNTQ" %%mm0, (%0, %%"REG_a", 2)\n\t"
2481                         MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2)\n\t"
2482                         MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2)\n\t"
2483                         MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2)\n\t"
2484                         "add $16, %%"REG_a"             \n\t"
2485                         "cmp %3, %%"REG_a"              \n\t"
2486                         " jb 1b                         \n\t"
2487                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2488                         : "memory", "%"REG_a
2489                 );
2490 #endif
2491                 for(w= (width&(~15)); w < width; w++)
2492                 {
2493                         dest[2*w+0] = src1[w];
2494                         dest[2*w+1] = src2[w];
2495                 }
2496 #else
2497                 for(w=0; w < width; w++)
2498                 {
2499                         dest[2*w+0] = src1[w];
2500                         dest[2*w+1] = src2[w];
2501                 }
2502 #endif
2503                 dest += dstStride;
2504                 src1 += src1Stride;
2505                 src2 += src2Stride;
2506         }
2507 #ifdef HAVE_MMX
2508         asm(
2509                 EMMS" \n\t"
2510                 SFENCE" \n\t"
2511                 ::: "memory"
2512                 );
2513 #endif
2514 }
2515
2516 static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
2517                         uint8_t *dst1, uint8_t *dst2,
2518                         long width, long height,
2519                         long srcStride1, long srcStride2,
2520                         long dstStride1, long dstStride2)
2521 {
2522     long y,x,w,h;
2523     w=width/2; h=height/2;
2524 #ifdef HAVE_MMX
2525     asm volatile(
2526         PREFETCH" %0\n\t"
2527         PREFETCH" %1\n\t"
2528         ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
2529 #endif
2530     for(y=0;y<h;y++){
2531         const uint8_t* s1=src1+srcStride1*(y>>1);
2532         uint8_t* d=dst1+dstStride1*y;
2533         x=0;
2534 #ifdef HAVE_MMX
2535         for(;x<w-31;x+=32)
2536         {
2537             asm volatile(
2538                 PREFETCH" 32%1\n\t"
2539                 "movq   %1, %%mm0\n\t"
2540                 "movq   8%1, %%mm2\n\t"
2541                 "movq   16%1, %%mm4\n\t"
2542                 "movq   24%1, %%mm6\n\t"
2543                 "movq   %%mm0, %%mm1\n\t"
2544                 "movq   %%mm2, %%mm3\n\t"
2545                 "movq   %%mm4, %%mm5\n\t"
2546                 "movq   %%mm6, %%mm7\n\t"
2547                 "punpcklbw %%mm0, %%mm0\n\t"
2548                 "punpckhbw %%mm1, %%mm1\n\t"
2549                 "punpcklbw %%mm2, %%mm2\n\t"
2550                 "punpckhbw %%mm3, %%mm3\n\t"
2551                 "punpcklbw %%mm4, %%mm4\n\t"
2552                 "punpckhbw %%mm5, %%mm5\n\t"
2553                 "punpcklbw %%mm6, %%mm6\n\t"
2554                 "punpckhbw %%mm7, %%mm7\n\t"
2555                 MOVNTQ" %%mm0, %0\n\t"
2556                 MOVNTQ" %%mm1, 8%0\n\t"
2557                 MOVNTQ" %%mm2, 16%0\n\t"
2558                 MOVNTQ" %%mm3, 24%0\n\t"
2559                 MOVNTQ" %%mm4, 32%0\n\t"
2560                 MOVNTQ" %%mm5, 40%0\n\t"
2561                 MOVNTQ" %%mm6, 48%0\n\t"
2562                 MOVNTQ" %%mm7, 56%0"
2563                 :"=m"(d[2*x])
2564                 :"m"(s1[x])
2565                 :"memory");
2566         }
2567 #endif
2568         for(;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
2569     }
2570     for(y=0;y<h;y++){
2571         const uint8_t* s2=src2+srcStride2*(y>>1);
2572         uint8_t* d=dst2+dstStride2*y;
2573         x=0;
2574 #ifdef HAVE_MMX
2575         for(;x<w-31;x+=32)
2576         {
2577             asm volatile(
2578                 PREFETCH" 32%1\n\t"
2579                 "movq   %1, %%mm0\n\t"
2580                 "movq   8%1, %%mm2\n\t"
2581                 "movq   16%1, %%mm4\n\t"
2582                 "movq   24%1, %%mm6\n\t"
2583                 "movq   %%mm0, %%mm1\n\t"
2584                 "movq   %%mm2, %%mm3\n\t"
2585                 "movq   %%mm4, %%mm5\n\t"
2586                 "movq   %%mm6, %%mm7\n\t"
2587                 "punpcklbw %%mm0, %%mm0\n\t"
2588                 "punpckhbw %%mm1, %%mm1\n\t"
2589                 "punpcklbw %%mm2, %%mm2\n\t"
2590                 "punpckhbw %%mm3, %%mm3\n\t"
2591                 "punpcklbw %%mm4, %%mm4\n\t"
2592                 "punpckhbw %%mm5, %%mm5\n\t"
2593                 "punpcklbw %%mm6, %%mm6\n\t"
2594                 "punpckhbw %%mm7, %%mm7\n\t"
2595                 MOVNTQ" %%mm0, %0\n\t"
2596                 MOVNTQ" %%mm1, 8%0\n\t"
2597                 MOVNTQ" %%mm2, 16%0\n\t"
2598                 MOVNTQ" %%mm3, 24%0\n\t"
2599                 MOVNTQ" %%mm4, 32%0\n\t"
2600                 MOVNTQ" %%mm5, 40%0\n\t"
2601                 MOVNTQ" %%mm6, 48%0\n\t"
2602                 MOVNTQ" %%mm7, 56%0"
2603                 :"=m"(d[2*x])
2604                 :"m"(s2[x])
2605                 :"memory");
2606         }
2607 #endif
2608         for(;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
2609     }
2610 #ifdef HAVE_MMX
2611         asm(
2612                 EMMS" \n\t"
2613                 SFENCE" \n\t"
2614                 ::: "memory"
2615                 );
2616 #endif
2617 }
2618
2619 static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
2620                         uint8_t *dst,
2621                         long width, long height,
2622                         long srcStride1, long srcStride2,
2623                         long srcStride3, long dstStride)
2624 {
2625     long y,x,w,h;
2626     w=width/2; h=height;
2627     for(y=0;y<h;y++){
2628         const uint8_t* yp=src1+srcStride1*y;
2629         const uint8_t* up=src2+srcStride2*(y>>2);
2630         const uint8_t* vp=src3+srcStride3*(y>>2);
2631         uint8_t* d=dst+dstStride*y;
2632         x=0;
2633 #ifdef HAVE_MMX
2634         for(;x<w-7;x+=8)
2635         {
2636             asm volatile(
2637                 PREFETCH" 32(%1, %0)\n\t"
2638                 PREFETCH" 32(%2, %0)\n\t"
2639                 PREFETCH" 32(%3, %0)\n\t"
2640                 "movq   (%1, %0, 4), %%mm0\n\t"       /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2641                 "movq   (%2, %0), %%mm1\n\t"       /* U0U1U2U3U4U5U6U7 */
2642                 "movq   (%3, %0), %%mm2\n\t"         /* V0V1V2V3V4V5V6V7 */
2643                 "movq   %%mm0, %%mm3\n\t"    /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2644                 "movq   %%mm1, %%mm4\n\t"    /* U0U1U2U3U4U5U6U7 */
2645                 "movq   %%mm2, %%mm5\n\t"    /* V0V1V2V3V4V5V6V7 */
2646                 "punpcklbw %%mm1, %%mm1\n\t" /* U0U0 U1U1 U2U2 U3U3 */
2647                 "punpcklbw %%mm2, %%mm2\n\t" /* V0V0 V1V1 V2V2 V3V3 */
2648                 "punpckhbw %%mm4, %%mm4\n\t" /* U4U4 U5U5 U6U6 U7U7 */
2649                 "punpckhbw %%mm5, %%mm5\n\t" /* V4V4 V5V5 V6V6 V7V7 */
2650
2651                 "movq   %%mm1, %%mm6\n\t"
2652                 "punpcklbw %%mm2, %%mm1\n\t" /* U0V0 U0V0 U1V1 U1V1*/
2653                 "punpcklbw %%mm1, %%mm0\n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
2654                 "punpckhbw %%mm1, %%mm3\n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
2655                 MOVNTQ" %%mm0, (%4, %0, 8)\n\t"
2656                 MOVNTQ" %%mm3, 8(%4, %0, 8)\n\t"
2657                 
2658                 "punpckhbw %%mm2, %%mm6\n\t" /* U2V2 U2V2 U3V3 U3V3*/
2659                 "movq   8(%1, %0, 4), %%mm0\n\t"
2660                 "movq   %%mm0, %%mm3\n\t"
2661                 "punpcklbw %%mm6, %%mm0\n\t" /* Y U2 Y V2 Y U2 Y V2*/
2662                 "punpckhbw %%mm6, %%mm3\n\t" /* Y U3 Y V3 Y U3 Y V3*/
2663                 MOVNTQ" %%mm0, 16(%4, %0, 8)\n\t"
2664                 MOVNTQ" %%mm3, 24(%4, %0, 8)\n\t"
2665
2666                 "movq   %%mm4, %%mm6\n\t"
2667                 "movq   16(%1, %0, 4), %%mm0\n\t"
2668                 "movq   %%mm0, %%mm3\n\t"
2669                 "punpcklbw %%mm5, %%mm4\n\t"
2670                 "punpcklbw %%mm4, %%mm0\n\t" /* Y U4 Y V4 Y U4 Y V4*/
2671                 "punpckhbw %%mm4, %%mm3\n\t" /* Y U5 Y V5 Y U5 Y V5*/
2672                 MOVNTQ" %%mm0, 32(%4, %0, 8)\n\t"
2673                 MOVNTQ" %%mm3, 40(%4, %0, 8)\n\t"
2674                 
2675                 "punpckhbw %%mm5, %%mm6\n\t"
2676                 "movq   24(%1, %0, 4), %%mm0\n\t"
2677                 "movq   %%mm0, %%mm3\n\t"
2678                 "punpcklbw %%mm6, %%mm0\n\t" /* Y U6 Y V6 Y U6 Y V6*/
2679                 "punpckhbw %%mm6, %%mm3\n\t" /* Y U7 Y V7 Y U7 Y V7*/
2680                 MOVNTQ" %%mm0, 48(%4, %0, 8)\n\t"
2681                 MOVNTQ" %%mm3, 56(%4, %0, 8)\n\t"
2682
2683                 : "+r" (x)
2684                 : "r"(yp), "r" (up), "r"(vp), "r"(d)
2685                 :"memory");
2686         }
2687 #endif
2688         for(; x<w; x++)
2689         {
2690             const long x2= x<<2;
2691             d[8*x+0]=yp[x2];
2692             d[8*x+1]=up[x];
2693             d[8*x+2]=yp[x2+1];
2694             d[8*x+3]=vp[x];
2695             d[8*x+4]=yp[x2+2];
2696             d[8*x+5]=up[x];
2697             d[8*x+6]=yp[x2+3];
2698             d[8*x+7]=vp[x];
2699         }
2700     }
2701 #ifdef HAVE_MMX
2702         asm(
2703                 EMMS" \n\t"
2704                 SFENCE" \n\t"
2705                 ::: "memory"
2706                 );
2707 #endif
2708 }
2709
2710 static inline void RENAME(rgb2rgb_init)(void){
2711         rgb15to16= RENAME(rgb15to16);
2712         rgb15to24= RENAME(rgb15to24);
2713         rgb15to32= RENAME(rgb15to32);
2714         rgb16to24= RENAME(rgb16to24);
2715         rgb16to32= RENAME(rgb16to32);
2716         rgb16to15= RENAME(rgb16to15);
2717         rgb24to16= RENAME(rgb24to16);
2718         rgb24to15= RENAME(rgb24to15);
2719         rgb24to32= RENAME(rgb24to32);
2720         rgb32to16= RENAME(rgb32to16);
2721         rgb32to15= RENAME(rgb32to15);
2722         rgb32to24= RENAME(rgb32to24);
2723         rgb24tobgr15= RENAME(rgb24tobgr15);
2724         rgb24tobgr16= RENAME(rgb24tobgr16);
2725         rgb24tobgr24= RENAME(rgb24tobgr24);
2726         rgb32tobgr32= RENAME(rgb32tobgr32);
2727         rgb32tobgr16= RENAME(rgb32tobgr16);
2728         rgb32tobgr15= RENAME(rgb32tobgr15);
2729         yv12toyuy2= RENAME(yv12toyuy2);
2730         yv12touyvy= RENAME(yv12touyvy);
2731         yuv422ptoyuy2= RENAME(yuv422ptoyuy2);
2732         yuy2toyv12= RENAME(yuy2toyv12);
2733 //      uyvytoyv12= RENAME(uyvytoyv12);
2734 //      yvu9toyv12= RENAME(yvu9toyv12);
2735         planar2x= RENAME(planar2x);
2736         rgb24toyv12= RENAME(rgb24toyv12);
2737         interleaveBytes= RENAME(interleaveBytes);
2738         vu9_to_vu12= RENAME(vu9_to_vu12);
2739         yvu9_to_yuy2= RENAME(yvu9_to_yuy2);
2740 }