skip MMX code in rgb32to15 if the size of the input is smaller than the
[ffmpeg.git] / libswscale / rgb2rgb_template.c
1 /*
2  *
3  *  rgb2rgb.c, Software RGB to RGB convertor
4  *  pluralize by Software PAL8 to RGB convertor
5  *               Software YUV to YUV convertor
6  *               Software YUV to RGB convertor
7  *  Written by Nick Kurshev.
8  *  palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
9  *  lot of big-endian byteorder fixes by Alex Beregszaszi
10  *
11  * This file is part of FFmpeg.
12  *
13  * FFmpeg is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  * FFmpeg is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21  * GNU General Public License for more details.
22  *
23  * You should have received a copy of the GNU General Public License
24  * along with FFmpeg; if not, write to the Free Software
25  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
26  * 
27  * the C code (not assembly, mmx, ...) of this file can be used
28  * under the LGPL license too
29  */
30
31 #include <stddef.h>
32 #include <inttypes.h> /* for __WORDSIZE */
33
34 #ifndef __WORDSIZE
35 // #warning You have misconfigured system and probably will lose performance!
36 #define __WORDSIZE MP_WORDSIZE
37 #endif
38
39 #undef PREFETCH
40 #undef MOVNTQ
41 #undef EMMS
42 #undef SFENCE
43 #undef MMREG_SIZE
44 #undef PREFETCHW
45 #undef PAVGB
46
47 #ifdef HAVE_SSE2
48 #define MMREG_SIZE 16
49 #else
50 #define MMREG_SIZE 8
51 #endif
52
53 #ifdef HAVE_3DNOW
54 #define PREFETCH  "prefetch"
55 #define PREFETCHW "prefetchw"
56 #define PAVGB     "pavgusb"
57 #elif defined ( HAVE_MMX2 )
58 #define PREFETCH "prefetchnta"
59 #define PREFETCHW "prefetcht0"
60 #define PAVGB     "pavgb"
61 #else
62 #ifdef __APPLE__
63 #define PREFETCH "#"
64 #define PREFETCHW "#"
65 #else
66 #define PREFETCH  " # nop"
67 #define PREFETCHW " # nop"
68 #endif
69 #endif
70
71 #ifdef HAVE_3DNOW
72 /* On K6 femms is faster of emms. On K7 femms is directly mapped on emms. */
73 #define EMMS     "femms"
74 #else
75 #define EMMS     "emms"
76 #endif
77
78 #ifdef HAVE_MMX2
79 #define MOVNTQ "movntq"
80 #define SFENCE "sfence"
81 #else
82 #define MOVNTQ "movq"
83 #define SFENCE " # nop"
84 #endif
85
86 static inline void RENAME(rgb24to32)(const uint8_t *src,uint8_t *dst,long src_size)
87 {
88   uint8_t *dest = dst;
89   const uint8_t *s = src;
90   const uint8_t *end;
91 #ifdef HAVE_MMX
92   const uint8_t *mm_end;
93 #endif
94   end = s + src_size;
95 #ifdef HAVE_MMX
96   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
97   mm_end = end - 23;
98   __asm __volatile("movq        %0, %%mm7"::"m"(mask32):"memory");
99   while(s < mm_end)
100   {
101     __asm __volatile(
102         PREFETCH"       32%1\n\t"
103         "movd   %1, %%mm0\n\t"
104         "punpckldq 3%1, %%mm0\n\t"
105         "movd   6%1, %%mm1\n\t"
106         "punpckldq 9%1, %%mm1\n\t"
107         "movd   12%1, %%mm2\n\t"
108         "punpckldq 15%1, %%mm2\n\t"
109         "movd   18%1, %%mm3\n\t"
110         "punpckldq 21%1, %%mm3\n\t"
111         "pand   %%mm7, %%mm0\n\t"
112         "pand   %%mm7, %%mm1\n\t"
113         "pand   %%mm7, %%mm2\n\t"
114         "pand   %%mm7, %%mm3\n\t"
115         MOVNTQ" %%mm0, %0\n\t"
116         MOVNTQ" %%mm1, 8%0\n\t"
117         MOVNTQ" %%mm2, 16%0\n\t"
118         MOVNTQ" %%mm3, 24%0"
119         :"=m"(*dest)
120         :"m"(*s)
121         :"memory");
122     dest += 32;
123     s += 24;
124   }
125   __asm __volatile(SFENCE:::"memory");
126   __asm __volatile(EMMS:::"memory");
127 #endif
128   while(s < end)
129   {
130 #ifdef WORDS_BIGENDIAN
131     /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */
132     *dest++ = 0;
133     *dest++ = s[2];
134     *dest++ = s[1];
135     *dest++ = s[0];
136     s+=3;
137 #else
138     *dest++ = *s++;
139     *dest++ = *s++;
140     *dest++ = *s++;
141     *dest++ = 0;
142 #endif
143   }
144 }
145
146 static inline void RENAME(rgb32to24)(const uint8_t *src,uint8_t *dst,long src_size)
147 {
148   uint8_t *dest = dst;
149   const uint8_t *s = src;
150   const uint8_t *end;
151 #ifdef HAVE_MMX
152   const uint8_t *mm_end;
153 #endif
154   end = s + src_size;
155 #ifdef HAVE_MMX
156   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
157   mm_end = end - 31;
158   while(s < mm_end)
159   {
160     __asm __volatile(
161         PREFETCH"       32%1\n\t"
162         "movq   %1, %%mm0\n\t"
163         "movq   8%1, %%mm1\n\t"
164         "movq   16%1, %%mm4\n\t"
165         "movq   24%1, %%mm5\n\t"
166         "movq   %%mm0, %%mm2\n\t"
167         "movq   %%mm1, %%mm3\n\t"
168         "movq   %%mm4, %%mm6\n\t"
169         "movq   %%mm5, %%mm7\n\t"
170         "psrlq  $8, %%mm2\n\t"
171         "psrlq  $8, %%mm3\n\t"
172         "psrlq  $8, %%mm6\n\t"
173         "psrlq  $8, %%mm7\n\t"
174         "pand   %2, %%mm0\n\t"
175         "pand   %2, %%mm1\n\t"
176         "pand   %2, %%mm4\n\t"
177         "pand   %2, %%mm5\n\t"
178         "pand   %3, %%mm2\n\t"
179         "pand   %3, %%mm3\n\t"
180         "pand   %3, %%mm6\n\t"
181         "pand   %3, %%mm7\n\t"
182         "por    %%mm2, %%mm0\n\t"
183         "por    %%mm3, %%mm1\n\t"
184         "por    %%mm6, %%mm4\n\t"
185         "por    %%mm7, %%mm5\n\t"
186
187         "movq   %%mm1, %%mm2\n\t"
188         "movq   %%mm4, %%mm3\n\t"
189         "psllq  $48, %%mm2\n\t"
190         "psllq  $32, %%mm3\n\t"
191         "pand   %4, %%mm2\n\t"
192         "pand   %5, %%mm3\n\t"
193         "por    %%mm2, %%mm0\n\t"
194         "psrlq  $16, %%mm1\n\t"
195         "psrlq  $32, %%mm4\n\t"
196         "psllq  $16, %%mm5\n\t"
197         "por    %%mm3, %%mm1\n\t"
198         "pand   %6, %%mm5\n\t"
199         "por    %%mm5, %%mm4\n\t"
200
201         MOVNTQ" %%mm0, %0\n\t"
202         MOVNTQ" %%mm1, 8%0\n\t"
203         MOVNTQ" %%mm4, 16%0"
204         :"=m"(*dest)
205         :"m"(*s),"m"(mask24l),
206          "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
207         :"memory");
208     dest += 24;
209     s += 32;
210   }
211   __asm __volatile(SFENCE:::"memory");
212   __asm __volatile(EMMS:::"memory");
213 #endif
214   while(s < end)
215   {
216 #ifdef WORDS_BIGENDIAN
217     /* RGB32 (= A,B,G,R) -> RGB24 (= R,G,B) */
218     s++;
219     dest[2] = *s++;
220     dest[1] = *s++;
221     dest[0] = *s++;
222     dest += 3;
223 #else
224     *dest++ = *s++;
225     *dest++ = *s++;
226     *dest++ = *s++;
227     s++;
228 #endif
229   }
230 }
231
232 /*
233  Original by Strepto/Astral
234  ported to gcc & bugfixed : A'rpi
235  MMX2, 3DNOW optimization by Nick Kurshev
236  32bit c version, and and&add trick by Michael Niedermayer
237 */
238 static inline void RENAME(rgb15to16)(const uint8_t *src,uint8_t *dst,long src_size)
239 {
240   register const uint8_t* s=src;
241   register uint8_t* d=dst;
242   register const uint8_t *end;
243   const uint8_t *mm_end;
244   end = s + src_size;
245 #ifdef HAVE_MMX
246   __asm __volatile(PREFETCH"    %0"::"m"(*s));
247   __asm __volatile("movq        %0, %%mm4"::"m"(mask15s));
248   mm_end = end - 15;
249   while(s<mm_end)
250   {
251         __asm __volatile(
252                 PREFETCH"       32%1\n\t"
253                 "movq   %1, %%mm0\n\t"
254                 "movq   8%1, %%mm2\n\t"
255                 "movq   %%mm0, %%mm1\n\t"
256                 "movq   %%mm2, %%mm3\n\t"
257                 "pand   %%mm4, %%mm0\n\t"
258                 "pand   %%mm4, %%mm2\n\t"
259                 "paddw  %%mm1, %%mm0\n\t"
260                 "paddw  %%mm3, %%mm2\n\t"
261                 MOVNTQ" %%mm0, %0\n\t"
262                 MOVNTQ" %%mm2, 8%0"
263                 :"=m"(*d)
264                 :"m"(*s)
265                 );
266         d+=16;
267         s+=16;
268   }
269   __asm __volatile(SFENCE:::"memory");
270   __asm __volatile(EMMS:::"memory");
271 #endif
272     mm_end = end - 3;
273     while(s < mm_end)
274     {
275         register unsigned x= *((uint32_t *)s);
276         *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
277         d+=4;
278         s+=4;
279     }
280     if(s < end)
281     {
282         register unsigned short x= *((uint16_t *)s);
283         *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
284     }
285 }
286
287 static inline void RENAME(rgb16to15)(const uint8_t *src,uint8_t *dst,long src_size)
288 {
289   register const uint8_t* s=src;
290   register uint8_t* d=dst;
291   register const uint8_t *end;
292   const uint8_t *mm_end;
293   end = s + src_size;
294 #ifdef HAVE_MMX
295   __asm __volatile(PREFETCH"    %0"::"m"(*s));
296   __asm __volatile("movq        %0, %%mm7"::"m"(mask15rg));
297   __asm __volatile("movq        %0, %%mm6"::"m"(mask15b));
298   mm_end = end - 15;
299   while(s<mm_end)
300   {
301         __asm __volatile(
302                 PREFETCH"       32%1\n\t"
303                 "movq   %1, %%mm0\n\t"
304                 "movq   8%1, %%mm2\n\t"
305                 "movq   %%mm0, %%mm1\n\t"
306                 "movq   %%mm2, %%mm3\n\t"
307                 "psrlq  $1, %%mm0\n\t"
308                 "psrlq  $1, %%mm2\n\t"
309                 "pand   %%mm7, %%mm0\n\t"
310                 "pand   %%mm7, %%mm2\n\t"
311                 "pand   %%mm6, %%mm1\n\t"
312                 "pand   %%mm6, %%mm3\n\t"
313                 "por    %%mm1, %%mm0\n\t"
314                 "por    %%mm3, %%mm2\n\t"
315                 MOVNTQ" %%mm0, %0\n\t"
316                 MOVNTQ" %%mm2, 8%0"
317                 :"=m"(*d)
318                 :"m"(*s)
319                 );
320         d+=16;
321         s+=16;
322   }
323   __asm __volatile(SFENCE:::"memory");
324   __asm __volatile(EMMS:::"memory");
325 #endif
326     mm_end = end - 3;
327     while(s < mm_end)
328     {
329         register uint32_t x= *((uint32_t *)s);
330         *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
331         s+=4;
332         d+=4;
333     }
334     if(s < end)
335     {
336         register uint16_t x= *((uint16_t *)s);
337         *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
338         s+=2;
339         d+=2;
340     }
341 }
342
343 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, long src_size)
344 {
345         const uint8_t *s = src;
346         const uint8_t *end;
347 #ifdef HAVE_MMX
348         const uint8_t *mm_end;
349 #endif
350         uint16_t *d = (uint16_t *)dst;
351         end = s + src_size;
352 #ifdef HAVE_MMX
353         mm_end = end - 15;
354 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
355         asm volatile(
356                 "movq %3, %%mm5                 \n\t"
357                 "movq %4, %%mm6                 \n\t"
358                 "movq %5, %%mm7                 \n\t"
359                 ASMALIGN(4)
360                 "1:                             \n\t"
361                 PREFETCH" 32(%1)                \n\t"
362                 "movd   (%1), %%mm0             \n\t"
363                 "movd   4(%1), %%mm3            \n\t"
364                 "punpckldq 8(%1), %%mm0         \n\t"
365                 "punpckldq 12(%1), %%mm3        \n\t"
366                 "movq %%mm0, %%mm1              \n\t"
367                 "movq %%mm3, %%mm4              \n\t"
368                 "pand %%mm6, %%mm0              \n\t"
369                 "pand %%mm6, %%mm3              \n\t"
370                 "pmaddwd %%mm7, %%mm0           \n\t"
371                 "pmaddwd %%mm7, %%mm3           \n\t"
372                 "pand %%mm5, %%mm1              \n\t"
373                 "pand %%mm5, %%mm4              \n\t"
374                 "por %%mm1, %%mm0               \n\t"   
375                 "por %%mm4, %%mm3               \n\t"
376                 "psrld $5, %%mm0                \n\t"
377                 "pslld $11, %%mm3               \n\t"
378                 "por %%mm3, %%mm0               \n\t"
379                 MOVNTQ" %%mm0, (%0)             \n\t"
380                 "add $16, %1                    \n\t"
381                 "add $8, %0                     \n\t"
382                 "cmp %2, %1                     \n\t"
383                 " jb 1b                         \n\t"
384                 : "+r" (d), "+r"(s)
385                 : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
386         );
387 #else
388         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
389         __asm __volatile(
390             "movq       %0, %%mm7\n\t"
391             "movq       %1, %%mm6\n\t"
392             ::"m"(red_16mask),"m"(green_16mask));
393         while(s < mm_end)
394         {
395             __asm __volatile(
396                 PREFETCH" 32%1\n\t"
397                 "movd   %1, %%mm0\n\t"
398                 "movd   4%1, %%mm3\n\t"
399                 "punpckldq 8%1, %%mm0\n\t"
400                 "punpckldq 12%1, %%mm3\n\t"
401                 "movq   %%mm0, %%mm1\n\t"
402                 "movq   %%mm0, %%mm2\n\t"
403                 "movq   %%mm3, %%mm4\n\t"
404                 "movq   %%mm3, %%mm5\n\t"
405                 "psrlq  $3, %%mm0\n\t"
406                 "psrlq  $3, %%mm3\n\t"
407                 "pand   %2, %%mm0\n\t"
408                 "pand   %2, %%mm3\n\t"
409                 "psrlq  $5, %%mm1\n\t"
410                 "psrlq  $5, %%mm4\n\t"
411                 "pand   %%mm6, %%mm1\n\t"
412                 "pand   %%mm6, %%mm4\n\t"
413                 "psrlq  $8, %%mm2\n\t"
414                 "psrlq  $8, %%mm5\n\t"
415                 "pand   %%mm7, %%mm2\n\t"
416                 "pand   %%mm7, %%mm5\n\t"
417                 "por    %%mm1, %%mm0\n\t"
418                 "por    %%mm4, %%mm3\n\t"
419                 "por    %%mm2, %%mm0\n\t"
420                 "por    %%mm5, %%mm3\n\t"
421                 "psllq  $16, %%mm3\n\t"
422                 "por    %%mm3, %%mm0\n\t"
423                 MOVNTQ" %%mm0, %0\n\t"
424                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
425                 d += 4;
426                 s += 16;
427         }
428 #endif
429         __asm __volatile(SFENCE:::"memory");
430         __asm __volatile(EMMS:::"memory");
431 #endif
432         while(s < end)
433         {
434                 register int rgb = *(uint32_t*)s; s += 4;
435                 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
436         }
437 }
438
439 static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
440 {
441         const uint8_t *s = src;
442         const uint8_t *end;
443 #ifdef HAVE_MMX
444         const uint8_t *mm_end;
445 #endif
446         uint16_t *d = (uint16_t *)dst;
447         end = s + src_size;
448 #ifdef HAVE_MMX
449         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
450         __asm __volatile(
451             "movq       %0, %%mm7\n\t"
452             "movq       %1, %%mm6\n\t"
453             ::"m"(red_16mask),"m"(green_16mask));
454         mm_end = end - 15;
455         while(s < mm_end)
456         {
457             __asm __volatile(
458                 PREFETCH" 32%1\n\t"
459                 "movd   %1, %%mm0\n\t"
460                 "movd   4%1, %%mm3\n\t"
461                 "punpckldq 8%1, %%mm0\n\t"
462                 "punpckldq 12%1, %%mm3\n\t"
463                 "movq   %%mm0, %%mm1\n\t"
464                 "movq   %%mm0, %%mm2\n\t"
465                 "movq   %%mm3, %%mm4\n\t"
466                 "movq   %%mm3, %%mm5\n\t"
467                 "psllq  $8, %%mm0\n\t"
468                 "psllq  $8, %%mm3\n\t"
469                 "pand   %%mm7, %%mm0\n\t"
470                 "pand   %%mm7, %%mm3\n\t"
471                 "psrlq  $5, %%mm1\n\t"
472                 "psrlq  $5, %%mm4\n\t"
473                 "pand   %%mm6, %%mm1\n\t"
474                 "pand   %%mm6, %%mm4\n\t"
475                 "psrlq  $19, %%mm2\n\t"
476                 "psrlq  $19, %%mm5\n\t"
477                 "pand   %2, %%mm2\n\t"
478                 "pand   %2, %%mm5\n\t"
479                 "por    %%mm1, %%mm0\n\t"
480                 "por    %%mm4, %%mm3\n\t"
481                 "por    %%mm2, %%mm0\n\t"
482                 "por    %%mm5, %%mm3\n\t"
483                 "psllq  $16, %%mm3\n\t"
484                 "por    %%mm3, %%mm0\n\t"
485                 MOVNTQ" %%mm0, %0\n\t"
486                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
487                 d += 4;
488                 s += 16;
489         }
490         __asm __volatile(SFENCE:::"memory");
491         __asm __volatile(EMMS:::"memory");
492 #endif
493         while(s < end)
494         {
495                 register int rgb = *(uint32_t*)s; s += 4;
496                 *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
497         }
498 }
499
500 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, long src_size)
501 {
502         const uint8_t *s = src;
503         const uint8_t *end;
504 #ifdef HAVE_MMX
505         const uint8_t *mm_end;
506 #endif
507         uint16_t *d = (uint16_t *)dst;
508         end = s + src_size;
509 #ifdef HAVE_MMX
510         mm_end = end - 15;
511 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
512         asm volatile(
513                 "movq %3, %%mm5                 \n\t"
514                 "movq %4, %%mm6                 \n\t"
515                 "movq %5, %%mm7                 \n\t"
516                 "jmp 2f                         \n\t"
517                 ASMALIGN(4)
518                 "1:                             \n\t"
519                 PREFETCH" 32(%1)                \n\t"
520                 "movd   (%1), %%mm0             \n\t"
521                 "movd   4(%1), %%mm3            \n\t"
522                 "punpckldq 8(%1), %%mm0         \n\t"
523                 "punpckldq 12(%1), %%mm3        \n\t"
524                 "movq %%mm0, %%mm1              \n\t"
525                 "movq %%mm3, %%mm4              \n\t"
526                 "pand %%mm6, %%mm0              \n\t"
527                 "pand %%mm6, %%mm3              \n\t"
528                 "pmaddwd %%mm7, %%mm0           \n\t"
529                 "pmaddwd %%mm7, %%mm3           \n\t"
530                 "pand %%mm5, %%mm1              \n\t"
531                 "pand %%mm5, %%mm4              \n\t"
532                 "por %%mm1, %%mm0               \n\t"   
533                 "por %%mm4, %%mm3               \n\t"
534                 "psrld $6, %%mm0                \n\t"
535                 "pslld $10, %%mm3               \n\t"
536                 "por %%mm3, %%mm0               \n\t"
537                 MOVNTQ" %%mm0, (%0)             \n\t"
538                 "add $16, %1                    \n\t"
539                 "add $8, %0                     \n\t"
540                 "2:                             \n\t"
541                 "cmp %2, %1                     \n\t"
542                 " jb 1b                         \n\t"
543                 : "+r" (d), "+r"(s)
544                 : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
545         );
546 #else
547         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
548         __asm __volatile(
549             "movq       %0, %%mm7\n\t"
550             "movq       %1, %%mm6\n\t"
551             ::"m"(red_15mask),"m"(green_15mask));
552         while(s < mm_end)
553         {
554             __asm __volatile(
555                 PREFETCH" 32%1\n\t"
556                 "movd   %1, %%mm0\n\t"
557                 "movd   4%1, %%mm3\n\t"
558                 "punpckldq 8%1, %%mm0\n\t"
559                 "punpckldq 12%1, %%mm3\n\t"
560                 "movq   %%mm0, %%mm1\n\t"
561                 "movq   %%mm0, %%mm2\n\t"
562                 "movq   %%mm3, %%mm4\n\t"
563                 "movq   %%mm3, %%mm5\n\t"
564                 "psrlq  $3, %%mm0\n\t"
565                 "psrlq  $3, %%mm3\n\t"
566                 "pand   %2, %%mm0\n\t"
567                 "pand   %2, %%mm3\n\t"
568                 "psrlq  $6, %%mm1\n\t"
569                 "psrlq  $6, %%mm4\n\t"
570                 "pand   %%mm6, %%mm1\n\t"
571                 "pand   %%mm6, %%mm4\n\t"
572                 "psrlq  $9, %%mm2\n\t"
573                 "psrlq  $9, %%mm5\n\t"
574                 "pand   %%mm7, %%mm2\n\t"
575                 "pand   %%mm7, %%mm5\n\t"
576                 "por    %%mm1, %%mm0\n\t"
577                 "por    %%mm4, %%mm3\n\t"
578                 "por    %%mm2, %%mm0\n\t"
579                 "por    %%mm5, %%mm3\n\t"
580                 "psllq  $16, %%mm3\n\t"
581                 "por    %%mm3, %%mm0\n\t"
582                 MOVNTQ" %%mm0, %0\n\t"
583                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
584                 d += 4;
585                 s += 16;
586         }
587 #endif
588         __asm __volatile(SFENCE:::"memory");
589         __asm __volatile(EMMS:::"memory");
590 #endif
591         while(s < end)
592         {
593                 register int rgb = *(uint32_t*)s; s += 4;
594                 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
595         }
596 }
597
598 static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
599 {
600         const uint8_t *s = src;
601         const uint8_t *end;
602 #ifdef HAVE_MMX
603         const uint8_t *mm_end;
604 #endif
605         uint16_t *d = (uint16_t *)dst;
606         end = s + src_size;
607 #ifdef HAVE_MMX
608         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
609         __asm __volatile(
610             "movq       %0, %%mm7\n\t"
611             "movq       %1, %%mm6\n\t"
612             ::"m"(red_15mask),"m"(green_15mask));
613         mm_end = end - 15;
614         while(s < mm_end)
615         {
616             __asm __volatile(
617                 PREFETCH" 32%1\n\t"
618                 "movd   %1, %%mm0\n\t"
619                 "movd   4%1, %%mm3\n\t"
620                 "punpckldq 8%1, %%mm0\n\t"
621                 "punpckldq 12%1, %%mm3\n\t"
622                 "movq   %%mm0, %%mm1\n\t"
623                 "movq   %%mm0, %%mm2\n\t"
624                 "movq   %%mm3, %%mm4\n\t"
625                 "movq   %%mm3, %%mm5\n\t"
626                 "psllq  $7, %%mm0\n\t"
627                 "psllq  $7, %%mm3\n\t"
628                 "pand   %%mm7, %%mm0\n\t"
629                 "pand   %%mm7, %%mm3\n\t"
630                 "psrlq  $6, %%mm1\n\t"
631                 "psrlq  $6, %%mm4\n\t"
632                 "pand   %%mm6, %%mm1\n\t"
633                 "pand   %%mm6, %%mm4\n\t"
634                 "psrlq  $19, %%mm2\n\t"
635                 "psrlq  $19, %%mm5\n\t"
636                 "pand   %2, %%mm2\n\t"
637                 "pand   %2, %%mm5\n\t"
638                 "por    %%mm1, %%mm0\n\t"
639                 "por    %%mm4, %%mm3\n\t"
640                 "por    %%mm2, %%mm0\n\t"
641                 "por    %%mm5, %%mm3\n\t"
642                 "psllq  $16, %%mm3\n\t"
643                 "por    %%mm3, %%mm0\n\t"
644                 MOVNTQ" %%mm0, %0\n\t"
645                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
646                 d += 4;
647                 s += 16;
648         }
649         __asm __volatile(SFENCE:::"memory");
650         __asm __volatile(EMMS:::"memory");
651 #endif
652         while(s < end)
653         {
654                 register int rgb = *(uint32_t*)s; s += 4;
655                 *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
656         }
657 }
658
659 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, long src_size)
660 {
661         const uint8_t *s = src;
662         const uint8_t *end;
663 #ifdef HAVE_MMX
664         const uint8_t *mm_end;
665 #endif
666         uint16_t *d = (uint16_t *)dst;
667         end = s + src_size;
668 #ifdef HAVE_MMX
669         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
670         __asm __volatile(
671             "movq       %0, %%mm7\n\t"
672             "movq       %1, %%mm6\n\t"
673             ::"m"(red_16mask),"m"(green_16mask));
674         mm_end = end - 11;
675         while(s < mm_end)
676         {
677             __asm __volatile(
678                 PREFETCH" 32%1\n\t"
679                 "movd   %1, %%mm0\n\t"
680                 "movd   3%1, %%mm3\n\t"
681                 "punpckldq 6%1, %%mm0\n\t"
682                 "punpckldq 9%1, %%mm3\n\t"
683                 "movq   %%mm0, %%mm1\n\t"
684                 "movq   %%mm0, %%mm2\n\t"
685                 "movq   %%mm3, %%mm4\n\t"
686                 "movq   %%mm3, %%mm5\n\t"
687                 "psrlq  $3, %%mm0\n\t"
688                 "psrlq  $3, %%mm3\n\t"
689                 "pand   %2, %%mm0\n\t"
690                 "pand   %2, %%mm3\n\t"
691                 "psrlq  $5, %%mm1\n\t"
692                 "psrlq  $5, %%mm4\n\t"
693                 "pand   %%mm6, %%mm1\n\t"
694                 "pand   %%mm6, %%mm4\n\t"
695                 "psrlq  $8, %%mm2\n\t"
696                 "psrlq  $8, %%mm5\n\t"
697                 "pand   %%mm7, %%mm2\n\t"
698                 "pand   %%mm7, %%mm5\n\t"
699                 "por    %%mm1, %%mm0\n\t"
700                 "por    %%mm4, %%mm3\n\t"
701                 "por    %%mm2, %%mm0\n\t"
702                 "por    %%mm5, %%mm3\n\t"
703                 "psllq  $16, %%mm3\n\t"
704                 "por    %%mm3, %%mm0\n\t"
705                 MOVNTQ" %%mm0, %0\n\t"
706                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
707                 d += 4;
708                 s += 12;
709         }
710         __asm __volatile(SFENCE:::"memory");
711         __asm __volatile(EMMS:::"memory");
712 #endif
713         while(s < end)
714         {
715                 const int b= *s++;
716                 const int g= *s++;
717                 const int r= *s++;
718                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
719         }
720 }
721
722 static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
723 {
724         const uint8_t *s = src;
725         const uint8_t *end;
726 #ifdef HAVE_MMX
727         const uint8_t *mm_end;
728 #endif
729         uint16_t *d = (uint16_t *)dst;
730         end = s + src_size;
731 #ifdef HAVE_MMX
732         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
733         __asm __volatile(
734             "movq       %0, %%mm7\n\t"
735             "movq       %1, %%mm6\n\t"
736             ::"m"(red_16mask),"m"(green_16mask));
737         mm_end = end - 15;
738         while(s < mm_end)
739         {
740             __asm __volatile(
741                 PREFETCH" 32%1\n\t"
742                 "movd   %1, %%mm0\n\t"
743                 "movd   3%1, %%mm3\n\t"
744                 "punpckldq 6%1, %%mm0\n\t"
745                 "punpckldq 9%1, %%mm3\n\t"
746                 "movq   %%mm0, %%mm1\n\t"
747                 "movq   %%mm0, %%mm2\n\t"
748                 "movq   %%mm3, %%mm4\n\t"
749                 "movq   %%mm3, %%mm5\n\t"
750                 "psllq  $8, %%mm0\n\t"
751                 "psllq  $8, %%mm3\n\t"
752                 "pand   %%mm7, %%mm0\n\t"
753                 "pand   %%mm7, %%mm3\n\t"
754                 "psrlq  $5, %%mm1\n\t"
755                 "psrlq  $5, %%mm4\n\t"
756                 "pand   %%mm6, %%mm1\n\t"
757                 "pand   %%mm6, %%mm4\n\t"
758                 "psrlq  $19, %%mm2\n\t"
759                 "psrlq  $19, %%mm5\n\t"
760                 "pand   %2, %%mm2\n\t"
761                 "pand   %2, %%mm5\n\t"
762                 "por    %%mm1, %%mm0\n\t"
763                 "por    %%mm4, %%mm3\n\t"
764                 "por    %%mm2, %%mm0\n\t"
765                 "por    %%mm5, %%mm3\n\t"
766                 "psllq  $16, %%mm3\n\t"
767                 "por    %%mm3, %%mm0\n\t"
768                 MOVNTQ" %%mm0, %0\n\t"
769                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
770                 d += 4;
771                 s += 12;
772         }
773         __asm __volatile(SFENCE:::"memory");
774         __asm __volatile(EMMS:::"memory");
775 #endif
776         while(s < end)
777         {
778                 const int r= *s++;
779                 const int g= *s++;
780                 const int b= *s++;
781                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
782         }
783 }
784
785 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, long src_size)
786 {
787         const uint8_t *s = src;
788         const uint8_t *end;
789 #ifdef HAVE_MMX
790         const uint8_t *mm_end;
791 #endif
792         uint16_t *d = (uint16_t *)dst;
793         end = s + src_size;
794 #ifdef HAVE_MMX
795         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
796         __asm __volatile(
797             "movq       %0, %%mm7\n\t"
798             "movq       %1, %%mm6\n\t"
799             ::"m"(red_15mask),"m"(green_15mask));
800         mm_end = end - 11;
801         while(s < mm_end)
802         {
803             __asm __volatile(
804                 PREFETCH" 32%1\n\t"
805                 "movd   %1, %%mm0\n\t"
806                 "movd   3%1, %%mm3\n\t"
807                 "punpckldq 6%1, %%mm0\n\t"
808                 "punpckldq 9%1, %%mm3\n\t"
809                 "movq   %%mm0, %%mm1\n\t"
810                 "movq   %%mm0, %%mm2\n\t"
811                 "movq   %%mm3, %%mm4\n\t"
812                 "movq   %%mm3, %%mm5\n\t"
813                 "psrlq  $3, %%mm0\n\t"
814                 "psrlq  $3, %%mm3\n\t"
815                 "pand   %2, %%mm0\n\t"
816                 "pand   %2, %%mm3\n\t"
817                 "psrlq  $6, %%mm1\n\t"
818                 "psrlq  $6, %%mm4\n\t"
819                 "pand   %%mm6, %%mm1\n\t"
820                 "pand   %%mm6, %%mm4\n\t"
821                 "psrlq  $9, %%mm2\n\t"
822                 "psrlq  $9, %%mm5\n\t"
823                 "pand   %%mm7, %%mm2\n\t"
824                 "pand   %%mm7, %%mm5\n\t"
825                 "por    %%mm1, %%mm0\n\t"
826                 "por    %%mm4, %%mm3\n\t"
827                 "por    %%mm2, %%mm0\n\t"
828                 "por    %%mm5, %%mm3\n\t"
829                 "psllq  $16, %%mm3\n\t"
830                 "por    %%mm3, %%mm0\n\t"
831                 MOVNTQ" %%mm0, %0\n\t"
832                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
833                 d += 4;
834                 s += 12;
835         }
836         __asm __volatile(SFENCE:::"memory");
837         __asm __volatile(EMMS:::"memory");
838 #endif
839         while(s < end)
840         {
841                 const int b= *s++;
842                 const int g= *s++;
843                 const int r= *s++;
844                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
845         }
846 }
847
848 static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
849 {
850         const uint8_t *s = src;
851         const uint8_t *end;
852 #ifdef HAVE_MMX
853         const uint8_t *mm_end;
854 #endif
855         uint16_t *d = (uint16_t *)dst;
856         end = s + src_size;
857 #ifdef HAVE_MMX
858         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
859         __asm __volatile(
860             "movq       %0, %%mm7\n\t"
861             "movq       %1, %%mm6\n\t"
862             ::"m"(red_15mask),"m"(green_15mask));
863         mm_end = end - 15;
864         while(s < mm_end)
865         {
866             __asm __volatile(
867                 PREFETCH" 32%1\n\t"
868                 "movd   %1, %%mm0\n\t"
869                 "movd   3%1, %%mm3\n\t"
870                 "punpckldq 6%1, %%mm0\n\t"
871                 "punpckldq 9%1, %%mm3\n\t"
872                 "movq   %%mm0, %%mm1\n\t"
873                 "movq   %%mm0, %%mm2\n\t"
874                 "movq   %%mm3, %%mm4\n\t"
875                 "movq   %%mm3, %%mm5\n\t"
876                 "psllq  $7, %%mm0\n\t"
877                 "psllq  $7, %%mm3\n\t"
878                 "pand   %%mm7, %%mm0\n\t"
879                 "pand   %%mm7, %%mm3\n\t"
880                 "psrlq  $6, %%mm1\n\t"
881                 "psrlq  $6, %%mm4\n\t"
882                 "pand   %%mm6, %%mm1\n\t"
883                 "pand   %%mm6, %%mm4\n\t"
884                 "psrlq  $19, %%mm2\n\t"
885                 "psrlq  $19, %%mm5\n\t"
886                 "pand   %2, %%mm2\n\t"
887                 "pand   %2, %%mm5\n\t"
888                 "por    %%mm1, %%mm0\n\t"
889                 "por    %%mm4, %%mm3\n\t"
890                 "por    %%mm2, %%mm0\n\t"
891                 "por    %%mm5, %%mm3\n\t"
892                 "psllq  $16, %%mm3\n\t"
893                 "por    %%mm3, %%mm0\n\t"
894                 MOVNTQ" %%mm0, %0\n\t"
895                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
896                 d += 4;
897                 s += 12;
898         }
899         __asm __volatile(SFENCE:::"memory");
900         __asm __volatile(EMMS:::"memory");
901 #endif
902         while(s < end)
903         {
904                 const int r= *s++;
905                 const int g= *s++;
906                 const int b= *s++;
907                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
908         }
909 }
910
911 /*
912   I use here less accurate approximation by simply
913  left-shifting the input
914   value and filling the low order bits with
915  zeroes. This method improves png's
916   compression but this scheme cannot reproduce white exactly, since it does not
917   generate an all-ones maximum value; the net effect is to darken the
918   image slightly.
919
920   The better method should be "left bit replication":
921
922    4 3 2 1 0
923    ---------
924    1 1 0 1 1
925
926    7 6 5 4 3  2 1 0
927    ----------------
928    1 1 0 1 1  1 1 0
929    |=======|  |===|
930        |      Leftmost Bits Repeated to Fill Open Bits
931        |
932    Original Bits
933 */
934 static inline void RENAME(rgb15to24)(const uint8_t *src, uint8_t *dst, long src_size)
935 {
936         const uint16_t *end;
937 #ifdef HAVE_MMX
938         const uint16_t *mm_end;
939 #endif
940         uint8_t *d = (uint8_t *)dst;
941         const uint16_t *s = (uint16_t *)src;
942         end = s + src_size/2;
943 #ifdef HAVE_MMX
944         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
945         mm_end = end - 7;
946         while(s < mm_end)
947         {
948             __asm __volatile(
949                 PREFETCH" 32%1\n\t"
950                 "movq   %1, %%mm0\n\t"
951                 "movq   %1, %%mm1\n\t"
952                 "movq   %1, %%mm2\n\t"
953                 "pand   %2, %%mm0\n\t"
954                 "pand   %3, %%mm1\n\t"
955                 "pand   %4, %%mm2\n\t"
956                 "psllq  $3, %%mm0\n\t"
957                 "psrlq  $2, %%mm1\n\t"
958                 "psrlq  $7, %%mm2\n\t"
959                 "movq   %%mm0, %%mm3\n\t"
960                 "movq   %%mm1, %%mm4\n\t"
961                 "movq   %%mm2, %%mm5\n\t"
962                 "punpcklwd %5, %%mm0\n\t"
963                 "punpcklwd %5, %%mm1\n\t"
964                 "punpcklwd %5, %%mm2\n\t"
965                 "punpckhwd %5, %%mm3\n\t"
966                 "punpckhwd %5, %%mm4\n\t"
967                 "punpckhwd %5, %%mm5\n\t"
968                 "psllq  $8, %%mm1\n\t"
969                 "psllq  $16, %%mm2\n\t"
970                 "por    %%mm1, %%mm0\n\t"
971                 "por    %%mm2, %%mm0\n\t"
972                 "psllq  $8, %%mm4\n\t"
973                 "psllq  $16, %%mm5\n\t"
974                 "por    %%mm4, %%mm3\n\t"
975                 "por    %%mm5, %%mm3\n\t"
976
977                 "movq   %%mm0, %%mm6\n\t"
978                 "movq   %%mm3, %%mm7\n\t"
979                 
980                 "movq   8%1, %%mm0\n\t"
981                 "movq   8%1, %%mm1\n\t"
982                 "movq   8%1, %%mm2\n\t"
983                 "pand   %2, %%mm0\n\t"
984                 "pand   %3, %%mm1\n\t"
985                 "pand   %4, %%mm2\n\t"
986                 "psllq  $3, %%mm0\n\t"
987                 "psrlq  $2, %%mm1\n\t"
988                 "psrlq  $7, %%mm2\n\t"
989                 "movq   %%mm0, %%mm3\n\t"
990                 "movq   %%mm1, %%mm4\n\t"
991                 "movq   %%mm2, %%mm5\n\t"
992                 "punpcklwd %5, %%mm0\n\t"
993                 "punpcklwd %5, %%mm1\n\t"
994                 "punpcklwd %5, %%mm2\n\t"
995                 "punpckhwd %5, %%mm3\n\t"
996                 "punpckhwd %5, %%mm4\n\t"
997                 "punpckhwd %5, %%mm5\n\t"
998                 "psllq  $8, %%mm1\n\t"
999                 "psllq  $16, %%mm2\n\t"
1000                 "por    %%mm1, %%mm0\n\t"
1001                 "por    %%mm2, %%mm0\n\t"
1002                 "psllq  $8, %%mm4\n\t"
1003                 "psllq  $16, %%mm5\n\t"
1004                 "por    %%mm4, %%mm3\n\t"
1005                 "por    %%mm5, %%mm3\n\t"
1006
1007                 :"=m"(*d)
1008                 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)
1009                 :"memory");
1010             /* Borrowed 32 to 24 */
1011             __asm __volatile(
1012                 "movq   %%mm0, %%mm4\n\t"
1013                 "movq   %%mm3, %%mm5\n\t"
1014                 "movq   %%mm6, %%mm0\n\t"
1015                 "movq   %%mm7, %%mm1\n\t"
1016                 
1017                 "movq   %%mm4, %%mm6\n\t"
1018                 "movq   %%mm5, %%mm7\n\t"
1019                 "movq   %%mm0, %%mm2\n\t"
1020                 "movq   %%mm1, %%mm3\n\t"
1021
1022                 "psrlq  $8, %%mm2\n\t"
1023                 "psrlq  $8, %%mm3\n\t"
1024                 "psrlq  $8, %%mm6\n\t"
1025                 "psrlq  $8, %%mm7\n\t"
1026                 "pand   %2, %%mm0\n\t"
1027                 "pand   %2, %%mm1\n\t"
1028                 "pand   %2, %%mm4\n\t"
1029                 "pand   %2, %%mm5\n\t"
1030                 "pand   %3, %%mm2\n\t"
1031                 "pand   %3, %%mm3\n\t"
1032                 "pand   %3, %%mm6\n\t"
1033                 "pand   %3, %%mm7\n\t"
1034                 "por    %%mm2, %%mm0\n\t"
1035                 "por    %%mm3, %%mm1\n\t"
1036                 "por    %%mm6, %%mm4\n\t"
1037                 "por    %%mm7, %%mm5\n\t"
1038
1039                 "movq   %%mm1, %%mm2\n\t"
1040                 "movq   %%mm4, %%mm3\n\t"
1041                 "psllq  $48, %%mm2\n\t"
1042                 "psllq  $32, %%mm3\n\t"
1043                 "pand   %4, %%mm2\n\t"
1044                 "pand   %5, %%mm3\n\t"
1045                 "por    %%mm2, %%mm0\n\t"
1046                 "psrlq  $16, %%mm1\n\t"
1047                 "psrlq  $32, %%mm4\n\t"
1048                 "psllq  $16, %%mm5\n\t"
1049                 "por    %%mm3, %%mm1\n\t"
1050                 "pand   %6, %%mm5\n\t"
1051                 "por    %%mm5, %%mm4\n\t"
1052
1053                 MOVNTQ" %%mm0, %0\n\t"
1054                 MOVNTQ" %%mm1, 8%0\n\t"
1055                 MOVNTQ" %%mm4, 16%0"
1056
1057                 :"=m"(*d)
1058                 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1059                 :"memory");
1060                 d += 24;
1061                 s += 8;
1062         }
1063         __asm __volatile(SFENCE:::"memory");
1064         __asm __volatile(EMMS:::"memory");
1065 #endif
1066         while(s < end)
1067         {
1068                 register uint16_t bgr;
1069                 bgr = *s++;
1070                 *d++ = (bgr&0x1F)<<3;
1071                 *d++ = (bgr&0x3E0)>>2;
1072                 *d++ = (bgr&0x7C00)>>7;
1073         }
1074 }
1075
1076 static inline void RENAME(rgb16to24)(const uint8_t *src, uint8_t *dst, long src_size)
1077 {
1078         const uint16_t *end;
1079 #ifdef HAVE_MMX
1080         const uint16_t *mm_end;
1081 #endif
1082         uint8_t *d = (uint8_t *)dst;
1083         const uint16_t *s = (const uint16_t *)src;
1084         end = s + src_size/2;
1085 #ifdef HAVE_MMX
1086         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1087         mm_end = end - 7;
1088         while(s < mm_end)
1089         {
1090             __asm __volatile(
1091                 PREFETCH" 32%1\n\t"
1092                 "movq   %1, %%mm0\n\t"
1093                 "movq   %1, %%mm1\n\t"
1094                 "movq   %1, %%mm2\n\t"
1095                 "pand   %2, %%mm0\n\t"
1096                 "pand   %3, %%mm1\n\t"
1097                 "pand   %4, %%mm2\n\t"
1098                 "psllq  $3, %%mm0\n\t"
1099                 "psrlq  $3, %%mm1\n\t"
1100                 "psrlq  $8, %%mm2\n\t"
1101                 "movq   %%mm0, %%mm3\n\t"
1102                 "movq   %%mm1, %%mm4\n\t"
1103                 "movq   %%mm2, %%mm5\n\t"
1104                 "punpcklwd %5, %%mm0\n\t"
1105                 "punpcklwd %5, %%mm1\n\t"
1106                 "punpcklwd %5, %%mm2\n\t"
1107                 "punpckhwd %5, %%mm3\n\t"
1108                 "punpckhwd %5, %%mm4\n\t"
1109                 "punpckhwd %5, %%mm5\n\t"
1110                 "psllq  $8, %%mm1\n\t"
1111                 "psllq  $16, %%mm2\n\t"
1112                 "por    %%mm1, %%mm0\n\t"
1113                 "por    %%mm2, %%mm0\n\t"
1114                 "psllq  $8, %%mm4\n\t"
1115                 "psllq  $16, %%mm5\n\t"
1116                 "por    %%mm4, %%mm3\n\t"
1117                 "por    %%mm5, %%mm3\n\t"
1118                 
1119                 "movq   %%mm0, %%mm6\n\t"
1120                 "movq   %%mm3, %%mm7\n\t"
1121
1122                 "movq   8%1, %%mm0\n\t"
1123                 "movq   8%1, %%mm1\n\t"
1124                 "movq   8%1, %%mm2\n\t"
1125                 "pand   %2, %%mm0\n\t"
1126                 "pand   %3, %%mm1\n\t"
1127                 "pand   %4, %%mm2\n\t"
1128                 "psllq  $3, %%mm0\n\t"
1129                 "psrlq  $3, %%mm1\n\t"
1130                 "psrlq  $8, %%mm2\n\t"
1131                 "movq   %%mm0, %%mm3\n\t"
1132                 "movq   %%mm1, %%mm4\n\t"
1133                 "movq   %%mm2, %%mm5\n\t"
1134                 "punpcklwd %5, %%mm0\n\t"
1135                 "punpcklwd %5, %%mm1\n\t"
1136                 "punpcklwd %5, %%mm2\n\t"
1137                 "punpckhwd %5, %%mm3\n\t"
1138                 "punpckhwd %5, %%mm4\n\t"
1139                 "punpckhwd %5, %%mm5\n\t"
1140                 "psllq  $8, %%mm1\n\t"
1141                 "psllq  $16, %%mm2\n\t"
1142                 "por    %%mm1, %%mm0\n\t"
1143                 "por    %%mm2, %%mm0\n\t"
1144                 "psllq  $8, %%mm4\n\t"
1145                 "psllq  $16, %%mm5\n\t"
1146                 "por    %%mm4, %%mm3\n\t"
1147                 "por    %%mm5, %%mm3\n\t"
1148                 :"=m"(*d)
1149                 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)           
1150                 :"memory");
1151             /* Borrowed 32 to 24 */
1152             __asm __volatile(
1153                 "movq   %%mm0, %%mm4\n\t"
1154                 "movq   %%mm3, %%mm5\n\t"
1155                 "movq   %%mm6, %%mm0\n\t"
1156                 "movq   %%mm7, %%mm1\n\t"
1157                 
1158                 "movq   %%mm4, %%mm6\n\t"
1159                 "movq   %%mm5, %%mm7\n\t"
1160                 "movq   %%mm0, %%mm2\n\t"
1161                 "movq   %%mm1, %%mm3\n\t"
1162
1163                 "psrlq  $8, %%mm2\n\t"
1164                 "psrlq  $8, %%mm3\n\t"
1165                 "psrlq  $8, %%mm6\n\t"
1166                 "psrlq  $8, %%mm7\n\t"
1167                 "pand   %2, %%mm0\n\t"
1168                 "pand   %2, %%mm1\n\t"
1169                 "pand   %2, %%mm4\n\t"
1170                 "pand   %2, %%mm5\n\t"
1171                 "pand   %3, %%mm2\n\t"
1172                 "pand   %3, %%mm3\n\t"
1173                 "pand   %3, %%mm6\n\t"
1174                 "pand   %3, %%mm7\n\t"
1175                 "por    %%mm2, %%mm0\n\t"
1176                 "por    %%mm3, %%mm1\n\t"
1177                 "por    %%mm6, %%mm4\n\t"
1178                 "por    %%mm7, %%mm5\n\t"
1179
1180                 "movq   %%mm1, %%mm2\n\t"
1181                 "movq   %%mm4, %%mm3\n\t"
1182                 "psllq  $48, %%mm2\n\t"
1183                 "psllq  $32, %%mm3\n\t"
1184                 "pand   %4, %%mm2\n\t"
1185                 "pand   %5, %%mm3\n\t"
1186                 "por    %%mm2, %%mm0\n\t"
1187                 "psrlq  $16, %%mm1\n\t"
1188                 "psrlq  $32, %%mm4\n\t"
1189                 "psllq  $16, %%mm5\n\t"
1190                 "por    %%mm3, %%mm1\n\t"
1191                 "pand   %6, %%mm5\n\t"
1192                 "por    %%mm5, %%mm4\n\t"
1193
1194                 MOVNTQ" %%mm0, %0\n\t"
1195                 MOVNTQ" %%mm1, 8%0\n\t"
1196                 MOVNTQ" %%mm4, 16%0"
1197
1198                 :"=m"(*d)
1199                 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1200                 :"memory");
1201                 d += 24;
1202                 s += 8;
1203         }
1204         __asm __volatile(SFENCE:::"memory");
1205         __asm __volatile(EMMS:::"memory");
1206 #endif
1207         while(s < end)
1208         {
1209                 register uint16_t bgr;
1210                 bgr = *s++;
1211                 *d++ = (bgr&0x1F)<<3;
1212                 *d++ = (bgr&0x7E0)>>3;
1213                 *d++ = (bgr&0xF800)>>8;
1214         }
1215 }
1216
1217 static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_size)
1218 {
1219         const uint16_t *end;
1220 #ifdef HAVE_MMX
1221         const uint16_t *mm_end;
1222 #endif
1223         uint8_t *d = (uint8_t *)dst;
1224         const uint16_t *s = (const uint16_t *)src;
1225         end = s + src_size/2;
1226 #ifdef HAVE_MMX
1227         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1228         __asm __volatile("pxor  %%mm7,%%mm7\n\t":::"memory");
1229         mm_end = end - 3;
1230         while(s < mm_end)
1231         {
1232             __asm __volatile(
1233                 PREFETCH" 32%1\n\t"
1234                 "movq   %1, %%mm0\n\t"
1235                 "movq   %1, %%mm1\n\t"
1236                 "movq   %1, %%mm2\n\t"
1237                 "pand   %2, %%mm0\n\t"
1238                 "pand   %3, %%mm1\n\t"
1239                 "pand   %4, %%mm2\n\t"
1240                 "psllq  $3, %%mm0\n\t"
1241                 "psrlq  $2, %%mm1\n\t"
1242                 "psrlq  $7, %%mm2\n\t"
1243                 "movq   %%mm0, %%mm3\n\t"
1244                 "movq   %%mm1, %%mm4\n\t"
1245                 "movq   %%mm2, %%mm5\n\t"
1246                 "punpcklwd %%mm7, %%mm0\n\t"
1247                 "punpcklwd %%mm7, %%mm1\n\t"
1248                 "punpcklwd %%mm7, %%mm2\n\t"
1249                 "punpckhwd %%mm7, %%mm3\n\t"
1250                 "punpckhwd %%mm7, %%mm4\n\t"
1251                 "punpckhwd %%mm7, %%mm5\n\t"
1252                 "psllq  $8, %%mm1\n\t"
1253                 "psllq  $16, %%mm2\n\t"
1254                 "por    %%mm1, %%mm0\n\t"
1255                 "por    %%mm2, %%mm0\n\t"
1256                 "psllq  $8, %%mm4\n\t"
1257                 "psllq  $16, %%mm5\n\t"
1258                 "por    %%mm4, %%mm3\n\t"
1259                 "por    %%mm5, %%mm3\n\t"
1260                 MOVNTQ" %%mm0, %0\n\t"
1261                 MOVNTQ" %%mm3, 8%0\n\t"
1262                 :"=m"(*d)
1263                 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r)
1264                 :"memory");
1265                 d += 16;
1266                 s += 4;
1267         }
1268         __asm __volatile(SFENCE:::"memory");
1269         __asm __volatile(EMMS:::"memory");
1270 #endif
1271         while(s < end)
1272         {
1273 #if 0 //slightly slower on athlon
1274                 int bgr= *s++;
1275                 *((uint32_t*)d)++ = ((bgr&0x1F)<<3) + ((bgr&0x3E0)<<6) + ((bgr&0x7C00)<<9);
1276 #else
1277                 register uint16_t bgr;
1278                 bgr = *s++;
1279 #ifdef WORDS_BIGENDIAN
1280                 *d++ = 0;
1281                 *d++ = (bgr&0x7C00)>>7;
1282                 *d++ = (bgr&0x3E0)>>2;
1283                 *d++ = (bgr&0x1F)<<3;
1284 #else
1285                 *d++ = (bgr&0x1F)<<3;
1286                 *d++ = (bgr&0x3E0)>>2;
1287                 *d++ = (bgr&0x7C00)>>7;
1288                 *d++ = 0;
1289 #endif
1290
1291 #endif
1292         }
1293 }
1294
1295 static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_size)
1296 {
1297         const uint16_t *end;
1298 #ifdef HAVE_MMX
1299         const uint16_t *mm_end;
1300 #endif
1301         uint8_t *d = (uint8_t *)dst;
1302         const uint16_t *s = (uint16_t *)src;
1303         end = s + src_size/2;
1304 #ifdef HAVE_MMX
1305         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1306         __asm __volatile("pxor  %%mm7,%%mm7\n\t":::"memory");
1307         mm_end = end - 3;
1308         while(s < mm_end)
1309         {
1310             __asm __volatile(
1311                 PREFETCH" 32%1\n\t"
1312                 "movq   %1, %%mm0\n\t"
1313                 "movq   %1, %%mm1\n\t"
1314                 "movq   %1, %%mm2\n\t"
1315                 "pand   %2, %%mm0\n\t"
1316                 "pand   %3, %%mm1\n\t"
1317                 "pand   %4, %%mm2\n\t"
1318                 "psllq  $3, %%mm0\n\t"
1319                 "psrlq  $3, %%mm1\n\t"
1320                 "psrlq  $8, %%mm2\n\t"
1321                 "movq   %%mm0, %%mm3\n\t"
1322                 "movq   %%mm1, %%mm4\n\t"
1323                 "movq   %%mm2, %%mm5\n\t"
1324                 "punpcklwd %%mm7, %%mm0\n\t"
1325                 "punpcklwd %%mm7, %%mm1\n\t"
1326                 "punpcklwd %%mm7, %%mm2\n\t"
1327                 "punpckhwd %%mm7, %%mm3\n\t"
1328                 "punpckhwd %%mm7, %%mm4\n\t"
1329                 "punpckhwd %%mm7, %%mm5\n\t"
1330                 "psllq  $8, %%mm1\n\t"
1331                 "psllq  $16, %%mm2\n\t"
1332                 "por    %%mm1, %%mm0\n\t"
1333                 "por    %%mm2, %%mm0\n\t"
1334                 "psllq  $8, %%mm4\n\t"
1335                 "psllq  $16, %%mm5\n\t"
1336                 "por    %%mm4, %%mm3\n\t"
1337                 "por    %%mm5, %%mm3\n\t"
1338                 MOVNTQ" %%mm0, %0\n\t"
1339                 MOVNTQ" %%mm3, 8%0\n\t"
1340                 :"=m"(*d)
1341                 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r)
1342                 :"memory");
1343                 d += 16;
1344                 s += 4;
1345         }
1346         __asm __volatile(SFENCE:::"memory");
1347         __asm __volatile(EMMS:::"memory");
1348 #endif
1349         while(s < end)
1350         {
1351                 register uint16_t bgr;
1352                 bgr = *s++;
1353 #ifdef WORDS_BIGENDIAN
1354                 *d++ = 0;
1355                 *d++ = (bgr&0xF800)>>8;
1356                 *d++ = (bgr&0x7E0)>>3;
1357                 *d++ = (bgr&0x1F)<<3;
1358 #else
1359                 *d++ = (bgr&0x1F)<<3;
1360                 *d++ = (bgr&0x7E0)>>3;
1361                 *d++ = (bgr&0xF800)>>8;
1362                 *d++ = 0;
1363 #endif
1364         }
1365 }
1366
1367 static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, long src_size)
1368 {
1369         long idx = 15 - src_size;
1370         uint8_t *s = (uint8_t *) src-idx, *d = dst-idx;
1371 #ifdef HAVE_MMX
1372         __asm __volatile(
1373                 "       test %0, %0                     \n"
1374                 "       jns 2f                          \n"
1375                 "       "PREFETCH" (%1, %0)             \n"
1376                 "       movq %3, %%mm7                  \n"
1377                 "       pxor %4, %%mm7                  \n"
1378                 "       movq %%mm7, %%mm6               \n"
1379                 "       pxor %5, %%mm7                  \n"
1380                         ASMALIGN(4)
1381                 "1:                                     \n"
1382                 "       "PREFETCH" 32(%1, %0)           \n"
1383                 "       movq (%1, %0), %%mm0            \n"
1384                 "       movq 8(%1, %0), %%mm1           \n"
1385 # ifdef HAVE_MMX2
1386                 "       pshufw $177, %%mm0, %%mm3       \n"
1387                 "       pshufw $177, %%mm1, %%mm5       \n"
1388                 "       pand %%mm7, %%mm0               \n"
1389                 "       pand %%mm6, %%mm3               \n"
1390                 "       pand %%mm7, %%mm1               \n"
1391                 "       pand %%mm6, %%mm5               \n"
1392                 "       por %%mm3, %%mm0                \n"
1393                 "       por %%mm5, %%mm1                \n"
1394 # else
1395                 "       movq %%mm0, %%mm2               \n"
1396                 "       movq %%mm1, %%mm4               \n"
1397                 "       pand %%mm7, %%mm0               \n"
1398                 "       pand %%mm6, %%mm2               \n"
1399                 "       pand %%mm7, %%mm1               \n"
1400                 "       pand %%mm6, %%mm4               \n"
1401                 "       movq %%mm2, %%mm3               \n"
1402                 "       movq %%mm4, %%mm5               \n"
1403                 "       pslld $16, %%mm2                \n"
1404                 "       psrld $16, %%mm3                \n"
1405                 "       pslld $16, %%mm4                \n"
1406                 "       psrld $16, %%mm5                \n"
1407                 "       por %%mm2, %%mm0                \n"
1408                 "       por %%mm4, %%mm1                \n"
1409                 "       por %%mm3, %%mm0                \n"
1410                 "       por %%mm5, %%mm1                \n"
1411 # endif
1412                 "       "MOVNTQ" %%mm0, (%2, %0)        \n"
1413                 "       "MOVNTQ" %%mm1, 8(%2, %0)       \n"
1414                 "       add $16, %0                     \n"
1415                 "       js 1b                           \n"
1416                 "       "SFENCE"                        \n"
1417                 "       "EMMS"                          \n"
1418                 "2:                                     \n"
1419                 : "+&r"(idx)
1420                 : "r" (s), "r" (d), "m" (mask32b), "m" (mask32r), "m" (mmx_one)
1421                 : "memory");
1422 #endif
1423         for (; idx<15; idx+=4) {
1424                 register int v = *(uint32_t *)&s[idx], g = v & 0xff00;
1425                 v &= 0xff00ff;
1426                 *(uint32_t *)&d[idx] = (v>>16) + g + (v<<16);
1427         }
1428 }
1429
1430 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, long src_size)
1431 {
1432         unsigned i;
1433 #ifdef HAVE_MMX
1434         long mmx_size= 23 - src_size;
1435         asm volatile (
1436                 "movq "MANGLE(mask24r)", %%mm5  \n\t"
1437                 "movq "MANGLE(mask24g)", %%mm6  \n\t"
1438                 "movq "MANGLE(mask24b)", %%mm7  \n\t"
1439                 ASMALIGN(4)
1440                 "1:                             \n\t"
1441                 PREFETCH" 32(%1, %%"REG_a")     \n\t"
1442                 "movq   (%1, %%"REG_a"), %%mm0  \n\t" // BGR BGR BG
1443                 "movq   (%1, %%"REG_a"), %%mm1  \n\t" // BGR BGR BG
1444                 "movq  2(%1, %%"REG_a"), %%mm2  \n\t" // R BGR BGR B
1445                 "psllq $16, %%mm0               \n\t" // 00 BGR BGR
1446                 "pand %%mm5, %%mm0              \n\t"
1447                 "pand %%mm6, %%mm1              \n\t"
1448                 "pand %%mm7, %%mm2              \n\t"
1449                 "por %%mm0, %%mm1               \n\t"
1450                 "por %%mm2, %%mm1               \n\t"                
1451                 "movq  6(%1, %%"REG_a"), %%mm0  \n\t" // BGR BGR BG
1452                 MOVNTQ" %%mm1,   (%2, %%"REG_a")\n\t" // RGB RGB RG
1453                 "movq  8(%1, %%"REG_a"), %%mm1  \n\t" // R BGR BGR B
1454                 "movq 10(%1, %%"REG_a"), %%mm2  \n\t" // GR BGR BGR
1455                 "pand %%mm7, %%mm0              \n\t"
1456                 "pand %%mm5, %%mm1              \n\t"
1457                 "pand %%mm6, %%mm2              \n\t"
1458                 "por %%mm0, %%mm1               \n\t"
1459                 "por %%mm2, %%mm1               \n\t"                
1460                 "movq 14(%1, %%"REG_a"), %%mm0  \n\t" // R BGR BGR B
1461                 MOVNTQ" %%mm1,  8(%2, %%"REG_a")\n\t" // B RGB RGB R
1462                 "movq 16(%1, %%"REG_a"), %%mm1  \n\t" // GR BGR BGR
1463                 "movq 18(%1, %%"REG_a"), %%mm2  \n\t" // BGR BGR BG
1464                 "pand %%mm6, %%mm0              \n\t"
1465                 "pand %%mm7, %%mm1              \n\t"
1466                 "pand %%mm5, %%mm2              \n\t"
1467                 "por %%mm0, %%mm1               \n\t"
1468                 "por %%mm2, %%mm1               \n\t"                
1469                 MOVNTQ" %%mm1, 16(%2, %%"REG_a")\n\t"
1470                 "add $24, %%"REG_a"             \n\t"
1471                 " js 1b                         \n\t"
1472                 : "+a" (mmx_size)
1473                 : "r" (src-mmx_size), "r"(dst-mmx_size)
1474         );
1475
1476         __asm __volatile(SFENCE:::"memory");
1477         __asm __volatile(EMMS:::"memory");
1478
1479         if(mmx_size==23) return; //finihsed, was multiple of 8
1480
1481         src+= src_size;
1482         dst+= src_size;
1483         src_size= 23-mmx_size;
1484         src-= src_size;
1485         dst-= src_size;
1486 #endif
1487         for(i=0; i<src_size; i+=3)
1488         {
1489                 register uint8_t x;
1490                 x          = src[i + 2];
1491                 dst[i + 1] = src[i + 1];
1492                 dst[i + 2] = src[i + 0];
1493                 dst[i + 0] = x;
1494         }
1495 }
1496
1497 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1498         long width, long height,
1499         long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1500 {
1501         long y;
1502         const long chromWidth= width>>1;
1503         for(y=0; y<height; y++)
1504         {
1505 #ifdef HAVE_MMX
1506 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1507                 asm volatile(
1508                         "xor %%"REG_a", %%"REG_a"       \n\t"
1509                         ASMALIGN(4)
1510                         "1:                             \n\t"
1511                         PREFETCH" 32(%1, %%"REG_a", 2)  \n\t"
1512                         PREFETCH" 32(%2, %%"REG_a")     \n\t"
1513                         PREFETCH" 32(%3, %%"REG_a")     \n\t"
1514                         "movq (%2, %%"REG_a"), %%mm0    \n\t" // U(0)
1515                         "movq %%mm0, %%mm2              \n\t" // U(0)
1516                         "movq (%3, %%"REG_a"), %%mm1    \n\t" // V(0)
1517                         "punpcklbw %%mm1, %%mm0         \n\t" // UVUV UVUV(0)
1518                         "punpckhbw %%mm1, %%mm2         \n\t" // UVUV UVUV(8)
1519
1520                         "movq (%1, %%"REG_a",2), %%mm3  \n\t" // Y(0)
1521                         "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1522                         "movq %%mm3, %%mm4              \n\t" // Y(0)
1523                         "movq %%mm5, %%mm6              \n\t" // Y(8)
1524                         "punpcklbw %%mm0, %%mm3         \n\t" // YUYV YUYV(0)
1525                         "punpckhbw %%mm0, %%mm4         \n\t" // YUYV YUYV(4)
1526                         "punpcklbw %%mm2, %%mm5         \n\t" // YUYV YUYV(8)
1527                         "punpckhbw %%mm2, %%mm6         \n\t" // YUYV YUYV(12)
1528
1529                         MOVNTQ" %%mm3, (%0, %%"REG_a", 4)\n\t"
1530                         MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1531                         MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4)\n\t"
1532                         MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1533
1534                         "add $8, %%"REG_a"              \n\t"
1535                         "cmp %4, %%"REG_a"              \n\t"
1536                         " jb 1b                         \n\t"
1537                         ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1538                         : "%"REG_a
1539                 );
1540 #else
1541
1542 #if defined ARCH_ALPHA && defined HAVE_MVI
1543 #define pl2yuy2(n)                                      \
1544         y1 = yc[n];                                     \
1545         y2 = yc2[n];                                    \
1546         u = uc[n];                                      \
1547         v = vc[n];                                      \
1548         asm("unpkbw %1, %0" : "=r"(y1) : "r"(y1));      \
1549         asm("unpkbw %1, %0" : "=r"(y2) : "r"(y2));      \
1550         asm("unpkbl %1, %0" : "=r"(u) : "r"(u));        \
1551         asm("unpkbl %1, %0" : "=r"(v) : "r"(v));        \
1552         yuv1 = (u << 8) + (v << 24);                    \
1553         yuv2 = yuv1 + y2;                               \
1554         yuv1 += y1;                                     \
1555         qdst[n] = yuv1;                                 \
1556         qdst2[n] = yuv2;
1557
1558                 int i;
1559                 uint64_t *qdst = (uint64_t *) dst;
1560                 uint64_t *qdst2 = (uint64_t *) (dst + dstStride);
1561                 const uint32_t *yc = (uint32_t *) ysrc;
1562                 const uint32_t *yc2 = (uint32_t *) (ysrc + lumStride);
1563                 const uint16_t *uc = (uint16_t*) usrc, *vc = (uint16_t*) vsrc;
1564                 for(i = 0; i < chromWidth; i += 8){
1565                         uint64_t y1, y2, yuv1, yuv2;
1566                         uint64_t u, v;
1567                         /* Prefetch */
1568                         asm("ldq $31,64(%0)" :: "r"(yc));
1569                         asm("ldq $31,64(%0)" :: "r"(yc2));
1570                         asm("ldq $31,64(%0)" :: "r"(uc));
1571                         asm("ldq $31,64(%0)" :: "r"(vc));
1572
1573                         pl2yuy2(0);
1574                         pl2yuy2(1);
1575                         pl2yuy2(2);
1576                         pl2yuy2(3);
1577
1578                         yc += 4;
1579                         yc2 += 4;
1580                         uc += 4;
1581                         vc += 4;
1582                         qdst += 4;
1583                         qdst2 += 4;
1584                 }
1585                 y++;
1586                 ysrc += lumStride;
1587                 dst += dstStride;
1588
1589 #elif __WORDSIZE >= 64
1590                 int i;
1591                 uint64_t *ldst = (uint64_t *) dst;
1592                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1593                 for(i = 0; i < chromWidth; i += 2){
1594                         uint64_t k, l;
1595                         k = yc[0] + (uc[0] << 8) +
1596                             (yc[1] << 16) + (vc[0] << 24);
1597                         l = yc[2] + (uc[1] << 8) +
1598                             (yc[3] << 16) + (vc[1] << 24);
1599                         *ldst++ = k + (l << 32);
1600                         yc += 4;
1601                         uc += 2;
1602                         vc += 2;
1603                 }
1604
1605 #else
1606                 int i, *idst = (int32_t *) dst;
1607                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1608                 for(i = 0; i < chromWidth; i++){
1609 #ifdef WORDS_BIGENDIAN
1610                         *idst++ = (yc[0] << 24)+ (uc[0] << 16) +
1611                             (yc[1] << 8) + (vc[0] << 0);
1612 #else
1613                         *idst++ = yc[0] + (uc[0] << 8) +
1614                             (yc[1] << 16) + (vc[0] << 24);
1615 #endif
1616                         yc += 2;
1617                         uc++;
1618                         vc++;
1619                 }
1620 #endif
1621 #endif
1622                 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1623                 {
1624                         usrc += chromStride;
1625                         vsrc += chromStride;
1626                 }
1627                 ysrc += lumStride;
1628                 dst += dstStride;
1629         }
1630 #ifdef HAVE_MMX
1631 asm(    EMMS" \n\t"
1632         SFENCE" \n\t"
1633         :::"memory");
1634 #endif
1635 }
1636
1637 /**
1638  *
1639  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1640  * problem for anyone then tell me, and ill fix it)
1641  */
1642 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1643         long width, long height,
1644         long lumStride, long chromStride, long dstStride)
1645 {
1646         //FIXME interpolate chroma
1647         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1648 }
1649
1650 static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1651         long width, long height,
1652         long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1653 {
1654         long y;
1655         const long chromWidth= width>>1;
1656         for(y=0; y<height; y++)
1657         {
1658 #ifdef HAVE_MMX
1659 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1660                 asm volatile(
1661                         "xor %%"REG_a", %%"REG_a"       \n\t"
1662                         ASMALIGN(4)
1663                         "1:                             \n\t"
1664                         PREFETCH" 32(%1, %%"REG_a", 2)  \n\t"
1665                         PREFETCH" 32(%2, %%"REG_a")     \n\t"
1666                         PREFETCH" 32(%3, %%"REG_a")     \n\t"
1667                         "movq (%2, %%"REG_a"), %%mm0    \n\t" // U(0)
1668                         "movq %%mm0, %%mm2              \n\t" // U(0)
1669                         "movq (%3, %%"REG_a"), %%mm1    \n\t" // V(0)
1670                         "punpcklbw %%mm1, %%mm0         \n\t" // UVUV UVUV(0)
1671                         "punpckhbw %%mm1, %%mm2         \n\t" // UVUV UVUV(8)
1672
1673                         "movq (%1, %%"REG_a",2), %%mm3  \n\t" // Y(0)
1674                         "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1675                         "movq %%mm0, %%mm4              \n\t" // Y(0)
1676                         "movq %%mm2, %%mm6              \n\t" // Y(8)
1677                         "punpcklbw %%mm3, %%mm0         \n\t" // YUYV YUYV(0)
1678                         "punpckhbw %%mm3, %%mm4         \n\t" // YUYV YUYV(4)
1679                         "punpcklbw %%mm5, %%mm2         \n\t" // YUYV YUYV(8)
1680                         "punpckhbw %%mm5, %%mm6         \n\t" // YUYV YUYV(12)
1681
1682                         MOVNTQ" %%mm0, (%0, %%"REG_a", 4)\n\t"
1683                         MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1684                         MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4)\n\t"
1685                         MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1686
1687                         "add $8, %%"REG_a"              \n\t"
1688                         "cmp %4, %%"REG_a"              \n\t"
1689                         " jb 1b                         \n\t"
1690                         ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1691                         : "%"REG_a
1692                 );
1693 #else
1694 //FIXME adapt the alpha asm code from yv12->yuy2
1695
1696 #if __WORDSIZE >= 64
1697                 int i;
1698                 uint64_t *ldst = (uint64_t *) dst;
1699                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1700                 for(i = 0; i < chromWidth; i += 2){
1701                         uint64_t k, l;
1702                         k = uc[0] + (yc[0] << 8) +
1703                             (vc[0] << 16) + (yc[1] << 24);
1704                         l = uc[1] + (yc[2] << 8) +
1705                             (vc[1] << 16) + (yc[3] << 24);
1706                         *ldst++ = k + (l << 32);
1707                         yc += 4;
1708                         uc += 2;
1709                         vc += 2;
1710                 }
1711
1712 #else
1713                 int i, *idst = (int32_t *) dst;
1714                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1715                 for(i = 0; i < chromWidth; i++){
1716 #ifdef WORDS_BIGENDIAN
1717                         *idst++ = (uc[0] << 24)+ (yc[0] << 16) +
1718                             (vc[0] << 8) + (yc[1] << 0);
1719 #else
1720                         *idst++ = uc[0] + (yc[0] << 8) +
1721                             (vc[0] << 16) + (yc[1] << 24);
1722 #endif
1723                         yc += 2;
1724                         uc++;
1725                         vc++;
1726                 }
1727 #endif
1728 #endif
1729                 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1730                 {
1731                         usrc += chromStride;
1732                         vsrc += chromStride;
1733                 }
1734                 ysrc += lumStride;
1735                 dst += dstStride;
1736         }
1737 #ifdef HAVE_MMX
1738 asm(    EMMS" \n\t"
1739         SFENCE" \n\t"
1740         :::"memory");
1741 #endif
1742 }
1743
1744 /**
1745  *
1746  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1747  * problem for anyone then tell me, and ill fix it)
1748  */
1749 static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1750         long width, long height,
1751         long lumStride, long chromStride, long dstStride)
1752 {
1753         //FIXME interpolate chroma
1754         RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1755 }
1756
1757 /**
1758  *
1759  * width should be a multiple of 16
1760  */
1761 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1762         long width, long height,
1763         long lumStride, long chromStride, long dstStride)
1764 {
1765         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1766 }
1767
1768 /**
1769  *
1770  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1771  * problem for anyone then tell me, and ill fix it)
1772  */
1773 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1774         long width, long height,
1775         long lumStride, long chromStride, long srcStride)
1776 {
1777         long y;
1778         const long chromWidth= width>>1;
1779         for(y=0; y<height; y+=2)
1780         {
1781 #ifdef HAVE_MMX
1782                 asm volatile(
1783                         "xor %%"REG_a", %%"REG_a"       \n\t"
1784                         "pcmpeqw %%mm7, %%mm7           \n\t"
1785                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
1786                         ASMALIGN(4)
1787                         "1:                             \n\t"
1788                         PREFETCH" 64(%0, %%"REG_a", 4)  \n\t"
1789                         "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1790                         "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1791                         "movq %%mm0, %%mm2              \n\t" // YUYV YUYV(0)
1792                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(4)
1793                         "psrlw $8, %%mm0                \n\t" // U0V0 U0V0(0)
1794                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(4)
1795                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(0)
1796                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(4)
1797                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
1798                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
1799
1800                         MOVNTQ" %%mm2, (%1, %%"REG_a", 2)\n\t"
1801
1802                         "movq 16(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(8)
1803                         "movq 24(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(12)
1804                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(8)
1805                         "movq %%mm2, %%mm4              \n\t" // YUYV YUYV(12)
1806                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(8)
1807                         "psrlw $8, %%mm2                \n\t" // U0V0 U0V0(12)
1808                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(8)
1809                         "pand %%mm7, %%mm4              \n\t" // Y0Y0 Y0Y0(12)
1810                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
1811                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
1812
1813                         MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2)\n\t"
1814
1815                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
1816                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
1817                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
1818                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
1819                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
1820                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
1821                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
1822                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
1823
1824                         MOVNTQ" %%mm0, (%3, %%"REG_a")  \n\t"
1825                         MOVNTQ" %%mm2, (%2, %%"REG_a")  \n\t"
1826
1827                         "add $8, %%"REG_a"              \n\t"
1828                         "cmp %4, %%"REG_a"              \n\t"
1829                         " jb 1b                         \n\t"
1830                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1831                         : "memory", "%"REG_a
1832                 );
1833
1834                 ydst += lumStride;
1835                 src  += srcStride;
1836
1837                 asm volatile(
1838                         "xor %%"REG_a", %%"REG_a"       \n\t"
1839                         ASMALIGN(4)
1840                         "1:                             \n\t"
1841                         PREFETCH" 64(%0, %%"REG_a", 4)  \n\t"
1842                         "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1843                         "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1844                         "movq 16(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(8)
1845                         "movq 24(%0, %%"REG_a", 4), %%mm3\n\t" // YUYV YUYV(12)
1846                         "pand %%mm7, %%mm0              \n\t" // Y0Y0 Y0Y0(0)
1847                         "pand %%mm7, %%mm1              \n\t" // Y0Y0 Y0Y0(4)
1848                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(8)
1849                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(12)
1850                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
1851                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
1852
1853                         MOVNTQ" %%mm0, (%1, %%"REG_a", 2)\n\t"
1854                         MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2)\n\t"
1855
1856                         "add $8, %%"REG_a"              \n\t"
1857                         "cmp %4, %%"REG_a"              \n\t"
1858                         " jb 1b                         \n\t"
1859
1860                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1861                         : "memory", "%"REG_a
1862                 );
1863 #else
1864                 long i;
1865                 for(i=0; i<chromWidth; i++)
1866                 {
1867                         ydst[2*i+0]     = src[4*i+0];
1868                         udst[i]         = src[4*i+1];
1869                         ydst[2*i+1]     = src[4*i+2];
1870                         vdst[i]         = src[4*i+3];
1871                 }
1872                 ydst += lumStride;
1873                 src  += srcStride;
1874
1875                 for(i=0; i<chromWidth; i++)
1876                 {
1877                         ydst[2*i+0]     = src[4*i+0];
1878                         ydst[2*i+1]     = src[4*i+2];
1879                 }
1880 #endif
1881                 udst += chromStride;
1882                 vdst += chromStride;
1883                 ydst += lumStride;
1884                 src  += srcStride;
1885         }
1886 #ifdef HAVE_MMX
1887 asm volatile(   EMMS" \n\t"
1888                 SFENCE" \n\t"
1889                 :::"memory");
1890 #endif
1891 }
1892
1893 static inline void RENAME(yvu9toyv12)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,
1894         uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1895         long width, long height, long lumStride, long chromStride)
1896 {
1897         /* Y Plane */
1898         memcpy(ydst, ysrc, width*height);
1899
1900         /* XXX: implement upscaling for U,V */
1901 }
1902
1903 static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, long srcWidth, long srcHeight, long srcStride, long dstStride)
1904 {
1905         long x,y;
1906         
1907         dst[0]= src[0];
1908         
1909         // first line
1910         for(x=0; x<srcWidth-1; x++){
1911                 dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
1912                 dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
1913         }
1914         dst[2*srcWidth-1]= src[srcWidth-1];
1915         
1916         dst+= dstStride;
1917
1918         for(y=1; y<srcHeight; y++){
1919 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
1920                 const long mmxSize= srcWidth&~15;
1921                 asm volatile(
1922                         "mov %4, %%"REG_a"              \n\t"
1923                         "1:                             \n\t"
1924                         "movq (%0, %%"REG_a"), %%mm0    \n\t"
1925                         "movq (%1, %%"REG_a"), %%mm1    \n\t"
1926                         "movq 1(%0, %%"REG_a"), %%mm2   \n\t"
1927                         "movq 1(%1, %%"REG_a"), %%mm3   \n\t"
1928                         "movq -1(%0, %%"REG_a"), %%mm4  \n\t"
1929                         "movq -1(%1, %%"REG_a"), %%mm5  \n\t"
1930                         PAVGB" %%mm0, %%mm5             \n\t"
1931                         PAVGB" %%mm0, %%mm3             \n\t"
1932                         PAVGB" %%mm0, %%mm5             \n\t"
1933                         PAVGB" %%mm0, %%mm3             \n\t"
1934                         PAVGB" %%mm1, %%mm4             \n\t"
1935                         PAVGB" %%mm1, %%mm2             \n\t"
1936                         PAVGB" %%mm1, %%mm4             \n\t"
1937                         PAVGB" %%mm1, %%mm2             \n\t"
1938                         "movq %%mm5, %%mm7              \n\t"
1939                         "movq %%mm4, %%mm6              \n\t"
1940                         "punpcklbw %%mm3, %%mm5         \n\t"
1941                         "punpckhbw %%mm3, %%mm7         \n\t"
1942                         "punpcklbw %%mm2, %%mm4         \n\t"
1943                         "punpckhbw %%mm2, %%mm6         \n\t"
1944 #if 1
1945                         MOVNTQ" %%mm5, (%2, %%"REG_a", 2)\n\t"
1946                         MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1947                         MOVNTQ" %%mm4, (%3, %%"REG_a", 2)\n\t"
1948                         MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1949 #else
1950                         "movq %%mm5, (%2, %%"REG_a", 2) \n\t"
1951                         "movq %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1952                         "movq %%mm4, (%3, %%"REG_a", 2) \n\t"
1953                         "movq %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1954 #endif
1955                         "add $8, %%"REG_a"              \n\t"
1956                         " js 1b                         \n\t"
1957                         :: "r" (src + mmxSize  ), "r" (src + srcStride + mmxSize  ),
1958                            "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
1959                            "g" (-mmxSize)
1960                         : "%"REG_a
1961
1962                 );
1963 #else
1964                 const long mmxSize=1;
1965 #endif
1966                 dst[0        ]= (3*src[0] +   src[srcStride])>>2;
1967                 dst[dstStride]= (  src[0] + 3*src[srcStride])>>2;
1968
1969                 for(x=mmxSize-1; x<srcWidth-1; x++){
1970                         dst[2*x          +1]= (3*src[x+0] +   src[x+srcStride+1])>>2;
1971                         dst[2*x+dstStride+2]= (  src[x+0] + 3*src[x+srcStride+1])>>2;
1972                         dst[2*x+dstStride+1]= (  src[x+1] + 3*src[x+srcStride  ])>>2;
1973                         dst[2*x          +2]= (3*src[x+1] +   src[x+srcStride  ])>>2;
1974                 }
1975                 dst[srcWidth*2 -1            ]= (3*src[srcWidth-1] +   src[srcWidth-1 + srcStride])>>2;
1976                 dst[srcWidth*2 -1 + dstStride]= (  src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
1977
1978                 dst+=dstStride*2;
1979                 src+=srcStride;
1980         }
1981         
1982         // last line
1983 #if 1
1984         dst[0]= src[0];
1985         
1986         for(x=0; x<srcWidth-1; x++){
1987                 dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
1988                 dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
1989         }
1990         dst[2*srcWidth-1]= src[srcWidth-1];
1991 #else
1992         for(x=0; x<srcWidth; x++){
1993                 dst[2*x+0]=
1994                 dst[2*x+1]= src[x];
1995         }
1996 #endif
1997
1998 #ifdef HAVE_MMX
1999 asm volatile(   EMMS" \n\t"
2000                 SFENCE" \n\t"
2001                 :::"memory");
2002 #endif
2003 }
2004
2005 /**
2006  *
2007  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
2008  * problem for anyone then tell me, and ill fix it)
2009  * chrominance data is only taken from every secound line others are ignored FIXME write HQ version
2010  */
2011 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2012         long width, long height,
2013         long lumStride, long chromStride, long srcStride)
2014 {
2015         long y;
2016         const long chromWidth= width>>1;
2017         for(y=0; y<height; y+=2)
2018         {
2019 #ifdef HAVE_MMX
2020                 asm volatile(
2021                         "xorl %%eax, %%eax              \n\t"
2022                         "pcmpeqw %%mm7, %%mm7           \n\t"
2023                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
2024                         ASMALIGN(4)
2025                         "1:                             \n\t"
2026                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
2027                         "movq (%0, %%eax, 4), %%mm0     \n\t" // UYVY UYVY(0)
2028                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // UYVY UYVY(4)
2029                         "movq %%mm0, %%mm2              \n\t" // UYVY UYVY(0)
2030                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(4)
2031                         "pand %%mm7, %%mm0              \n\t" // U0V0 U0V0(0)
2032                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(4)
2033                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(0)
2034                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(4)
2035                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
2036                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
2037
2038                         MOVNTQ" %%mm2, (%1, %%eax, 2)   \n\t"
2039
2040                         "movq 16(%0, %%eax, 4), %%mm1   \n\t" // UYVY UYVY(8)
2041                         "movq 24(%0, %%eax, 4), %%mm2   \n\t" // UYVY UYVY(12)
2042                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(8)
2043                         "movq %%mm2, %%mm4              \n\t" // UYVY UYVY(12)
2044                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(8)
2045                         "pand %%mm7, %%mm2              \n\t" // U0V0 U0V0(12)
2046                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(8)
2047                         "psrlw $8, %%mm4                \n\t" // Y0Y0 Y0Y0(12)
2048                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
2049                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
2050
2051                         MOVNTQ" %%mm3, 8(%1, %%eax, 2)  \n\t"
2052
2053                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
2054                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
2055                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
2056                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
2057                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
2058                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
2059                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
2060                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
2061
2062                         MOVNTQ" %%mm0, (%3, %%eax)      \n\t"
2063                         MOVNTQ" %%mm2, (%2, %%eax)      \n\t"
2064
2065                         "addl $8, %%eax                 \n\t"
2066                         "cmpl %4, %%eax                 \n\t"
2067                         " jb 1b                         \n\t"
2068                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2069                         : "memory", "%eax"
2070                 );
2071
2072                 ydst += lumStride;
2073                 src  += srcStride;
2074
2075                 asm volatile(
2076                         "xorl %%eax, %%eax              \n\t"
2077                         ASMALIGN(4)
2078                         "1:                             \n\t"
2079                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
2080                         "movq (%0, %%eax, 4), %%mm0     \n\t" // YUYV YUYV(0)
2081                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // YUYV YUYV(4)
2082                         "movq 16(%0, %%eax, 4), %%mm2   \n\t" // YUYV YUYV(8)
2083                         "movq 24(%0, %%eax, 4), %%mm3   \n\t" // YUYV YUYV(12)
2084                         "psrlw $8, %%mm0                \n\t" // Y0Y0 Y0Y0(0)
2085                         "psrlw $8, %%mm1                \n\t" // Y0Y0 Y0Y0(4)
2086                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(8)
2087                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(12)
2088                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
2089                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
2090
2091                         MOVNTQ" %%mm0, (%1, %%eax, 2)   \n\t"
2092                         MOVNTQ" %%mm2, 8(%1, %%eax, 2)  \n\t"
2093
2094                         "addl $8, %%eax                 \n\t"
2095                         "cmpl %4, %%eax                 \n\t"
2096                         " jb 1b                         \n\t"
2097
2098                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2099                         : "memory", "%eax"
2100                 );
2101 #else
2102                 long i;
2103                 for(i=0; i<chromWidth; i++)
2104                 {
2105                         udst[i]         = src[4*i+0];
2106                         ydst[2*i+0]     = src[4*i+1];
2107                         vdst[i]         = src[4*i+2];
2108                         ydst[2*i+1]     = src[4*i+3];
2109                 }
2110                 ydst += lumStride;
2111                 src  += srcStride;
2112
2113                 for(i=0; i<chromWidth; i++)
2114                 {
2115                         ydst[2*i+0]     = src[4*i+1];
2116                         ydst[2*i+1]     = src[4*i+3];
2117                 }
2118 #endif
2119                 udst += chromStride;
2120                 vdst += chromStride;
2121                 ydst += lumStride;
2122                 src  += srcStride;
2123         }
2124 #ifdef HAVE_MMX
2125 asm volatile(   EMMS" \n\t"
2126                 SFENCE" \n\t"
2127                 :::"memory");
2128 #endif
2129 }
2130
2131 /**
2132  *
2133  * height should be a multiple of 2 and width should be a multiple of 2 (if this is a
2134  * problem for anyone then tell me, and ill fix it)
2135  * chrominance data is only taken from every secound line others are ignored in the C version FIXME write HQ version
2136  */
2137 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2138         long width, long height,
2139         long lumStride, long chromStride, long srcStride)
2140 {
2141         long y;
2142         const long chromWidth= width>>1;
2143 #ifdef HAVE_MMX
2144         for(y=0; y<height-2; y+=2)
2145         {
2146                 long i;
2147                 for(i=0; i<2; i++)
2148                 {
2149                         asm volatile(
2150                                 "mov %2, %%"REG_a"              \n\t"
2151                                 "movq "MANGLE(bgr2YCoeff)", %%mm6               \n\t"
2152                                 "movq "MANGLE(w1111)", %%mm5            \n\t"
2153                                 "pxor %%mm7, %%mm7              \n\t"
2154                                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2155                                 ASMALIGN(4)
2156                                 "1:                             \n\t"
2157                                 PREFETCH" 64(%0, %%"REG_d")     \n\t"
2158                                 "movd (%0, %%"REG_d"), %%mm0    \n\t"
2159                                 "movd 3(%0, %%"REG_d"), %%mm1   \n\t"
2160                                 "punpcklbw %%mm7, %%mm0         \n\t"
2161                                 "punpcklbw %%mm7, %%mm1         \n\t"
2162                                 "movd 6(%0, %%"REG_d"), %%mm2   \n\t"
2163                                 "movd 9(%0, %%"REG_d"), %%mm3   \n\t"
2164                                 "punpcklbw %%mm7, %%mm2         \n\t"
2165                                 "punpcklbw %%mm7, %%mm3         \n\t"
2166                                 "pmaddwd %%mm6, %%mm0           \n\t"
2167                                 "pmaddwd %%mm6, %%mm1           \n\t"
2168                                 "pmaddwd %%mm6, %%mm2           \n\t"
2169                                 "pmaddwd %%mm6, %%mm3           \n\t"
2170 #ifndef FAST_BGR2YV12
2171                                 "psrad $8, %%mm0                \n\t"
2172                                 "psrad $8, %%mm1                \n\t"
2173                                 "psrad $8, %%mm2                \n\t"
2174                                 "psrad $8, %%mm3                \n\t"
2175 #endif
2176                                 "packssdw %%mm1, %%mm0          \n\t"
2177                                 "packssdw %%mm3, %%mm2          \n\t"
2178                                 "pmaddwd %%mm5, %%mm0           \n\t"
2179                                 "pmaddwd %%mm5, %%mm2           \n\t"
2180                                 "packssdw %%mm2, %%mm0          \n\t"
2181                                 "psraw $7, %%mm0                \n\t"
2182
2183                                 "movd 12(%0, %%"REG_d"), %%mm4  \n\t"
2184                                 "movd 15(%0, %%"REG_d"), %%mm1  \n\t"
2185                                 "punpcklbw %%mm7, %%mm4         \n\t"
2186                                 "punpcklbw %%mm7, %%mm1         \n\t"
2187                                 "movd 18(%0, %%"REG_d"), %%mm2  \n\t"
2188                                 "movd 21(%0, %%"REG_d"), %%mm3  \n\t"
2189                                 "punpcklbw %%mm7, %%mm2         \n\t"
2190                                 "punpcklbw %%mm7, %%mm3         \n\t"
2191                                 "pmaddwd %%mm6, %%mm4           \n\t"
2192                                 "pmaddwd %%mm6, %%mm1           \n\t"
2193                                 "pmaddwd %%mm6, %%mm2           \n\t"
2194                                 "pmaddwd %%mm6, %%mm3           \n\t"
2195 #ifndef FAST_BGR2YV12
2196                                 "psrad $8, %%mm4                \n\t"
2197                                 "psrad $8, %%mm1                \n\t"
2198                                 "psrad $8, %%mm2                \n\t"
2199                                 "psrad $8, %%mm3                \n\t"
2200 #endif
2201                                 "packssdw %%mm1, %%mm4          \n\t"
2202                                 "packssdw %%mm3, %%mm2          \n\t"
2203                                 "pmaddwd %%mm5, %%mm4           \n\t"
2204                                 "pmaddwd %%mm5, %%mm2           \n\t"
2205                                 "add $24, %%"REG_d"             \n\t"
2206                                 "packssdw %%mm2, %%mm4          \n\t"
2207                                 "psraw $7, %%mm4                \n\t"
2208
2209                                 "packuswb %%mm4, %%mm0          \n\t"
2210                                 "paddusb "MANGLE(bgr2YOffset)", %%mm0   \n\t"
2211
2212                                 MOVNTQ" %%mm0, (%1, %%"REG_a")  \n\t"
2213                                 "add $8, %%"REG_a"              \n\t"
2214                                 " js 1b                         \n\t"
2215                                 : : "r" (src+width*3), "r" (ydst+width), "g" (-width)
2216                                 : "%"REG_a, "%"REG_d
2217                         );
2218                         ydst += lumStride;
2219                         src  += srcStride;
2220                 }
2221                 src -= srcStride*2;
2222                 asm volatile(
2223                         "mov %4, %%"REG_a"              \n\t"
2224                         "movq "MANGLE(w1111)", %%mm5            \n\t"
2225                         "movq "MANGLE(bgr2UCoeff)", %%mm6               \n\t"
2226                         "pxor %%mm7, %%mm7              \n\t"
2227                         "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2228                         "add %%"REG_d", %%"REG_d"       \n\t"
2229                         ASMALIGN(4)
2230                         "1:                             \n\t"
2231                         PREFETCH" 64(%0, %%"REG_d")     \n\t"
2232                         PREFETCH" 64(%1, %%"REG_d")     \n\t"
2233 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2234                         "movq (%0, %%"REG_d"), %%mm0    \n\t"
2235                         "movq (%1, %%"REG_d"), %%mm1    \n\t"
2236                         "movq 6(%0, %%"REG_d"), %%mm2   \n\t"
2237                         "movq 6(%1, %%"REG_d"), %%mm3   \n\t"
2238                         PAVGB" %%mm1, %%mm0             \n\t"
2239                         PAVGB" %%mm3, %%mm2             \n\t"
2240                         "movq %%mm0, %%mm1              \n\t"
2241                         "movq %%mm2, %%mm3              \n\t"
2242                         "psrlq $24, %%mm0               \n\t"
2243                         "psrlq $24, %%mm2               \n\t"
2244                         PAVGB" %%mm1, %%mm0             \n\t"
2245                         PAVGB" %%mm3, %%mm2             \n\t"
2246                         "punpcklbw %%mm7, %%mm0         \n\t"
2247                         "punpcklbw %%mm7, %%mm2         \n\t"
2248 #else
2249                         "movd (%0, %%"REG_d"), %%mm0    \n\t"
2250                         "movd (%1, %%"REG_d"), %%mm1    \n\t"
2251                         "movd 3(%0, %%"REG_d"), %%mm2   \n\t"
2252                         "movd 3(%1, %%"REG_d"), %%mm3   \n\t"
2253                         "punpcklbw %%mm7, %%mm0         \n\t"
2254                         "punpcklbw %%mm7, %%mm1         \n\t"
2255                         "punpcklbw %%mm7, %%mm2         \n\t"
2256                         "punpcklbw %%mm7, %%mm3         \n\t"
2257                         "paddw %%mm1, %%mm0             \n\t"
2258                         "paddw %%mm3, %%mm2             \n\t"
2259                         "paddw %%mm2, %%mm0             \n\t"
2260                         "movd 6(%0, %%"REG_d"), %%mm4   \n\t"
2261                         "movd 6(%1, %%"REG_d"), %%mm1   \n\t"
2262                         "movd 9(%0, %%"REG_d"), %%mm2   \n\t"
2263                         "movd 9(%1, %%"REG_d"), %%mm3   \n\t"
2264                         "punpcklbw %%mm7, %%mm4         \n\t"
2265                         "punpcklbw %%mm7, %%mm1         \n\t"
2266                         "punpcklbw %%mm7, %%mm2         \n\t"
2267                         "punpcklbw %%mm7, %%mm3         \n\t"
2268                         "paddw %%mm1, %%mm4             \n\t"
2269                         "paddw %%mm3, %%mm2             \n\t"
2270                         "paddw %%mm4, %%mm2             \n\t"
2271                         "psrlw $2, %%mm0                \n\t"
2272                         "psrlw $2, %%mm2                \n\t"
2273 #endif
2274                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
2275                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
2276
2277                         "pmaddwd %%mm0, %%mm1           \n\t"
2278                         "pmaddwd %%mm2, %%mm3           \n\t"
2279                         "pmaddwd %%mm6, %%mm0           \n\t"
2280                         "pmaddwd %%mm6, %%mm2           \n\t"
2281 #ifndef FAST_BGR2YV12
2282                         "psrad $8, %%mm0                \n\t"
2283                         "psrad $8, %%mm1                \n\t"
2284                         "psrad $8, %%mm2                \n\t"
2285                         "psrad $8, %%mm3                \n\t"
2286 #endif
2287                         "packssdw %%mm2, %%mm0          \n\t"
2288                         "packssdw %%mm3, %%mm1          \n\t"
2289                         "pmaddwd %%mm5, %%mm0           \n\t"
2290                         "pmaddwd %%mm5, %%mm1           \n\t"
2291                         "packssdw %%mm1, %%mm0          \n\t" // V1 V0 U1 U0
2292                         "psraw $7, %%mm0                \n\t"
2293
2294 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2295                         "movq 12(%0, %%"REG_d"), %%mm4  \n\t"
2296                         "movq 12(%1, %%"REG_d"), %%mm1  \n\t"
2297                         "movq 18(%0, %%"REG_d"), %%mm2  \n\t"
2298                         "movq 18(%1, %%"REG_d"), %%mm3  \n\t"
2299                         PAVGB" %%mm1, %%mm4             \n\t"
2300                         PAVGB" %%mm3, %%mm2             \n\t"
2301                         "movq %%mm4, %%mm1              \n\t"
2302                         "movq %%mm2, %%mm3              \n\t"
2303                         "psrlq $24, %%mm4               \n\t"
2304                         "psrlq $24, %%mm2               \n\t"
2305                         PAVGB" %%mm1, %%mm4             \n\t"
2306                         PAVGB" %%mm3, %%mm2             \n\t"
2307                         "punpcklbw %%mm7, %%mm4         \n\t"
2308                         "punpcklbw %%mm7, %%mm2         \n\t"
2309 #else
2310                         "movd 12(%0, %%"REG_d"), %%mm4  \n\t"
2311                         "movd 12(%1, %%"REG_d"), %%mm1  \n\t"
2312                         "movd 15(%0, %%"REG_d"), %%mm2  \n\t"
2313                         "movd 15(%1, %%"REG_d"), %%mm3  \n\t"
2314                         "punpcklbw %%mm7, %%mm4         \n\t"
2315                         "punpcklbw %%mm7, %%mm1         \n\t"
2316                         "punpcklbw %%mm7, %%mm2         \n\t"
2317                         "punpcklbw %%mm7, %%mm3         \n\t"
2318                         "paddw %%mm1, %%mm4             \n\t"
2319                         "paddw %%mm3, %%mm2             \n\t"
2320                         "paddw %%mm2, %%mm4             \n\t"
2321                         "movd 18(%0, %%"REG_d"), %%mm5  \n\t"
2322                         "movd 18(%1, %%"REG_d"), %%mm1  \n\t"
2323                         "movd 21(%0, %%"REG_d"), %%mm2  \n\t"
2324                         "movd 21(%1, %%"REG_d"), %%mm3  \n\t"
2325                         "punpcklbw %%mm7, %%mm5         \n\t"
2326                         "punpcklbw %%mm7, %%mm1         \n\t"
2327                         "punpcklbw %%mm7, %%mm2         \n\t"
2328                         "punpcklbw %%mm7, %%mm3         \n\t"
2329                         "paddw %%mm1, %%mm5             \n\t"
2330                         "paddw %%mm3, %%mm2             \n\t"
2331                         "paddw %%mm5, %%mm2             \n\t"
2332                         "movq "MANGLE(w1111)", %%mm5            \n\t"
2333                         "psrlw $2, %%mm4                \n\t"
2334                         "psrlw $2, %%mm2                \n\t"
2335 #endif
2336                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
2337                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
2338
2339                         "pmaddwd %%mm4, %%mm1           \n\t"
2340                         "pmaddwd %%mm2, %%mm3           \n\t"
2341                         "pmaddwd %%mm6, %%mm4           \n\t"
2342                         "pmaddwd %%mm6, %%mm2           \n\t"
2343 #ifndef FAST_BGR2YV12
2344                         "psrad $8, %%mm4                \n\t"
2345                         "psrad $8, %%mm1                \n\t"
2346                         "psrad $8, %%mm2                \n\t"
2347                         "psrad $8, %%mm3                \n\t"
2348 #endif
2349                         "packssdw %%mm2, %%mm4          \n\t"
2350                         "packssdw %%mm3, %%mm1          \n\t"
2351                         "pmaddwd %%mm5, %%mm4           \n\t"
2352                         "pmaddwd %%mm5, %%mm1           \n\t"
2353                         "add $24, %%"REG_d"             \n\t"
2354                         "packssdw %%mm1, %%mm4          \n\t" // V3 V2 U3 U2
2355                         "psraw $7, %%mm4                \n\t"
2356
2357                         "movq %%mm0, %%mm1              \n\t"
2358                         "punpckldq %%mm4, %%mm0         \n\t"
2359                         "punpckhdq %%mm4, %%mm1         \n\t"
2360                         "packsswb %%mm1, %%mm0          \n\t"
2361                         "paddb "MANGLE(bgr2UVOffset)", %%mm0    \n\t"
2362                         "movd %%mm0, (%2, %%"REG_a")    \n\t"
2363                         "punpckhdq %%mm0, %%mm0         \n\t"
2364                         "movd %%mm0, (%3, %%"REG_a")    \n\t"
2365                         "add $4, %%"REG_a"              \n\t"
2366                         " js 1b                         \n\t"
2367                         : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth)
2368                         : "%"REG_a, "%"REG_d
2369                 );
2370
2371                 udst += chromStride;
2372                 vdst += chromStride;
2373                 src  += srcStride*2;
2374         }
2375
2376         asm volatile(   EMMS" \n\t"
2377                         SFENCE" \n\t"
2378                         :::"memory");
2379 #else
2380         y=0;
2381 #endif
2382         for(; y<height; y+=2)
2383         {
2384                 long i;
2385                 for(i=0; i<chromWidth; i++)
2386                 {
2387                         unsigned int b= src[6*i+0];
2388                         unsigned int g= src[6*i+1];
2389                         unsigned int r= src[6*i+2];
2390
2391                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2392                         unsigned int V  =  ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128;
2393                         unsigned int U  =  ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128;
2394
2395                         udst[i]         = U;
2396                         vdst[i]         = V;
2397                         ydst[2*i]       = Y;
2398
2399                         b= src[6*i+3];
2400                         g= src[6*i+4];
2401                         r= src[6*i+5];
2402
2403                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2404                         ydst[2*i+1]     = Y;
2405                 }
2406                 ydst += lumStride;
2407                 src  += srcStride;
2408
2409                 for(i=0; i<chromWidth; i++)
2410                 {
2411                         unsigned int b= src[6*i+0];
2412                         unsigned int g= src[6*i+1];
2413                         unsigned int r= src[6*i+2];
2414
2415                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2416
2417                         ydst[2*i]       = Y;
2418
2419                         b= src[6*i+3];
2420                         g= src[6*i+4];
2421                         r= src[6*i+5];
2422
2423                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2424                         ydst[2*i+1]     = Y;
2425                 }
2426                 udst += chromStride;
2427                 vdst += chromStride;
2428                 ydst += lumStride;
2429                 src  += srcStride;
2430         }
2431 }
2432
2433 void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest,
2434                             long width, long height, long src1Stride,
2435                             long src2Stride, long dstStride){
2436         long h;
2437
2438         for(h=0; h < height; h++)
2439         {
2440                 long w;
2441
2442 #ifdef HAVE_MMX
2443 #ifdef HAVE_SSE2
2444                 asm(
2445                         "xor %%"REG_a", %%"REG_a"       \n\t"
2446                         "1:                             \n\t"
2447                         PREFETCH" 64(%1, %%"REG_a")     \n\t"
2448                         PREFETCH" 64(%2, %%"REG_a")     \n\t"
2449                         "movdqa (%1, %%"REG_a"), %%xmm0 \n\t"
2450                         "movdqa (%1, %%"REG_a"), %%xmm1 \n\t"
2451                         "movdqa (%2, %%"REG_a"), %%xmm2 \n\t"
2452                         "punpcklbw %%xmm2, %%xmm0       \n\t"
2453                         "punpckhbw %%xmm2, %%xmm1       \n\t"
2454                         "movntdq %%xmm0, (%0, %%"REG_a", 2)\n\t"
2455                         "movntdq %%xmm1, 16(%0, %%"REG_a", 2)\n\t"
2456                         "add $16, %%"REG_a"             \n\t"
2457                         "cmp %3, %%"REG_a"              \n\t"
2458                         " jb 1b                         \n\t"
2459                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2460                         : "memory", "%"REG_a""
2461                 );
2462 #else
2463                 asm(
2464                         "xor %%"REG_a", %%"REG_a"       \n\t"
2465                         "1:                             \n\t"
2466                         PREFETCH" 64(%1, %%"REG_a")     \n\t"
2467                         PREFETCH" 64(%2, %%"REG_a")     \n\t"
2468                         "movq (%1, %%"REG_a"), %%mm0    \n\t"
2469                         "movq 8(%1, %%"REG_a"), %%mm2   \n\t"
2470                         "movq %%mm0, %%mm1              \n\t"
2471                         "movq %%mm2, %%mm3              \n\t"
2472                         "movq (%2, %%"REG_a"), %%mm4    \n\t"
2473                         "movq 8(%2, %%"REG_a"), %%mm5   \n\t"
2474                         "punpcklbw %%mm4, %%mm0         \n\t"
2475                         "punpckhbw %%mm4, %%mm1         \n\t"
2476                         "punpcklbw %%mm5, %%mm2         \n\t"
2477                         "punpckhbw %%mm5, %%mm3         \n\t"
2478                         MOVNTQ" %%mm0, (%0, %%"REG_a", 2)\n\t"
2479                         MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2)\n\t"
2480                         MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2)\n\t"
2481                         MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2)\n\t"
2482                         "add $16, %%"REG_a"             \n\t"
2483                         "cmp %3, %%"REG_a"              \n\t"
2484                         " jb 1b                         \n\t"
2485                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2486                         : "memory", "%"REG_a
2487                 );
2488 #endif
2489                 for(w= (width&(~15)); w < width; w++)
2490                 {
2491                         dest[2*w+0] = src1[w];
2492                         dest[2*w+1] = src2[w];
2493                 }
2494 #else
2495                 for(w=0; w < width; w++)
2496                 {
2497                         dest[2*w+0] = src1[w];
2498                         dest[2*w+1] = src2[w];
2499                 }
2500 #endif
2501                 dest += dstStride;
2502                 src1 += src1Stride;
2503                 src2 += src2Stride;
2504         }
2505 #ifdef HAVE_MMX
2506         asm(
2507                 EMMS" \n\t"
2508                 SFENCE" \n\t"
2509                 ::: "memory"
2510                 );
2511 #endif
2512 }
2513
2514 static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
2515                         uint8_t *dst1, uint8_t *dst2,
2516                         long width, long height,
2517                         long srcStride1, long srcStride2,
2518                         long dstStride1, long dstStride2)
2519 {
2520     long y,x,w,h;
2521     w=width/2; h=height/2;
2522 #ifdef HAVE_MMX
2523     asm volatile(
2524         PREFETCH" %0\n\t"
2525         PREFETCH" %1\n\t"
2526         ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
2527 #endif
2528     for(y=0;y<h;y++){
2529         const uint8_t* s1=src1+srcStride1*(y>>1);
2530         uint8_t* d=dst1+dstStride1*y;
2531         x=0;
2532 #ifdef HAVE_MMX
2533         for(;x<w-31;x+=32)
2534         {
2535             asm volatile(
2536                 PREFETCH" 32%1\n\t"
2537                 "movq   %1, %%mm0\n\t"
2538                 "movq   8%1, %%mm2\n\t"
2539                 "movq   16%1, %%mm4\n\t"
2540                 "movq   24%1, %%mm6\n\t"
2541                 "movq   %%mm0, %%mm1\n\t"
2542                 "movq   %%mm2, %%mm3\n\t"
2543                 "movq   %%mm4, %%mm5\n\t"
2544                 "movq   %%mm6, %%mm7\n\t"
2545                 "punpcklbw %%mm0, %%mm0\n\t"
2546                 "punpckhbw %%mm1, %%mm1\n\t"
2547                 "punpcklbw %%mm2, %%mm2\n\t"
2548                 "punpckhbw %%mm3, %%mm3\n\t"
2549                 "punpcklbw %%mm4, %%mm4\n\t"
2550                 "punpckhbw %%mm5, %%mm5\n\t"
2551                 "punpcklbw %%mm6, %%mm6\n\t"
2552                 "punpckhbw %%mm7, %%mm7\n\t"
2553                 MOVNTQ" %%mm0, %0\n\t"
2554                 MOVNTQ" %%mm1, 8%0\n\t"
2555                 MOVNTQ" %%mm2, 16%0\n\t"
2556                 MOVNTQ" %%mm3, 24%0\n\t"
2557                 MOVNTQ" %%mm4, 32%0\n\t"
2558                 MOVNTQ" %%mm5, 40%0\n\t"
2559                 MOVNTQ" %%mm6, 48%0\n\t"
2560                 MOVNTQ" %%mm7, 56%0"
2561                 :"=m"(d[2*x])
2562                 :"m"(s1[x])
2563                 :"memory");
2564         }
2565 #endif
2566         for(;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
2567     }
2568     for(y=0;y<h;y++){
2569         const uint8_t* s2=src2+srcStride2*(y>>1);
2570         uint8_t* d=dst2+dstStride2*y;
2571         x=0;
2572 #ifdef HAVE_MMX
2573         for(;x<w-31;x+=32)
2574         {
2575             asm volatile(
2576                 PREFETCH" 32%1\n\t"
2577                 "movq   %1, %%mm0\n\t"
2578                 "movq   8%1, %%mm2\n\t"
2579                 "movq   16%1, %%mm4\n\t"
2580                 "movq   24%1, %%mm6\n\t"
2581                 "movq   %%mm0, %%mm1\n\t"
2582                 "movq   %%mm2, %%mm3\n\t"
2583                 "movq   %%mm4, %%mm5\n\t"
2584                 "movq   %%mm6, %%mm7\n\t"
2585                 "punpcklbw %%mm0, %%mm0\n\t"
2586                 "punpckhbw %%mm1, %%mm1\n\t"
2587                 "punpcklbw %%mm2, %%mm2\n\t"
2588                 "punpckhbw %%mm3, %%mm3\n\t"
2589                 "punpcklbw %%mm4, %%mm4\n\t"
2590                 "punpckhbw %%mm5, %%mm5\n\t"
2591                 "punpcklbw %%mm6, %%mm6\n\t"
2592                 "punpckhbw %%mm7, %%mm7\n\t"
2593                 MOVNTQ" %%mm0, %0\n\t"
2594                 MOVNTQ" %%mm1, 8%0\n\t"
2595                 MOVNTQ" %%mm2, 16%0\n\t"
2596                 MOVNTQ" %%mm3, 24%0\n\t"
2597                 MOVNTQ" %%mm4, 32%0\n\t"
2598                 MOVNTQ" %%mm5, 40%0\n\t"
2599                 MOVNTQ" %%mm6, 48%0\n\t"
2600                 MOVNTQ" %%mm7, 56%0"
2601                 :"=m"(d[2*x])
2602                 :"m"(s2[x])
2603                 :"memory");
2604         }
2605 #endif
2606         for(;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
2607     }
2608 #ifdef HAVE_MMX
2609         asm(
2610                 EMMS" \n\t"
2611                 SFENCE" \n\t"
2612                 ::: "memory"
2613                 );
2614 #endif
2615 }
2616
2617 static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
2618                         uint8_t *dst,
2619                         long width, long height,
2620                         long srcStride1, long srcStride2,
2621                         long srcStride3, long dstStride)
2622 {
2623     long y,x,w,h;
2624     w=width/2; h=height;
2625     for(y=0;y<h;y++){
2626         const uint8_t* yp=src1+srcStride1*y;
2627         const uint8_t* up=src2+srcStride2*(y>>2);
2628         const uint8_t* vp=src3+srcStride3*(y>>2);
2629         uint8_t* d=dst+dstStride*y;
2630         x=0;
2631 #ifdef HAVE_MMX
2632         for(;x<w-7;x+=8)
2633         {
2634             asm volatile(
2635                 PREFETCH" 32(%1, %0)\n\t"
2636                 PREFETCH" 32(%2, %0)\n\t"
2637                 PREFETCH" 32(%3, %0)\n\t"
2638                 "movq   (%1, %0, 4), %%mm0\n\t"       /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2639                 "movq   (%2, %0), %%mm1\n\t"       /* U0U1U2U3U4U5U6U7 */
2640                 "movq   (%3, %0), %%mm2\n\t"         /* V0V1V2V3V4V5V6V7 */
2641                 "movq   %%mm0, %%mm3\n\t"    /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2642                 "movq   %%mm1, %%mm4\n\t"    /* U0U1U2U3U4U5U6U7 */
2643                 "movq   %%mm2, %%mm5\n\t"    /* V0V1V2V3V4V5V6V7 */
2644                 "punpcklbw %%mm1, %%mm1\n\t" /* U0U0 U1U1 U2U2 U3U3 */
2645                 "punpcklbw %%mm2, %%mm2\n\t" /* V0V0 V1V1 V2V2 V3V3 */
2646                 "punpckhbw %%mm4, %%mm4\n\t" /* U4U4 U5U5 U6U6 U7U7 */
2647                 "punpckhbw %%mm5, %%mm5\n\t" /* V4V4 V5V5 V6V6 V7V7 */
2648
2649                 "movq   %%mm1, %%mm6\n\t"
2650                 "punpcklbw %%mm2, %%mm1\n\t" /* U0V0 U0V0 U1V1 U1V1*/
2651                 "punpcklbw %%mm1, %%mm0\n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
2652                 "punpckhbw %%mm1, %%mm3\n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
2653                 MOVNTQ" %%mm0, (%4, %0, 8)\n\t"
2654                 MOVNTQ" %%mm3, 8(%4, %0, 8)\n\t"
2655                 
2656                 "punpckhbw %%mm2, %%mm6\n\t" /* U2V2 U2V2 U3V3 U3V3*/
2657                 "movq   8(%1, %0, 4), %%mm0\n\t"
2658                 "movq   %%mm0, %%mm3\n\t"
2659                 "punpcklbw %%mm6, %%mm0\n\t" /* Y U2 Y V2 Y U2 Y V2*/
2660                 "punpckhbw %%mm6, %%mm3\n\t" /* Y U3 Y V3 Y U3 Y V3*/
2661                 MOVNTQ" %%mm0, 16(%4, %0, 8)\n\t"
2662                 MOVNTQ" %%mm3, 24(%4, %0, 8)\n\t"
2663
2664                 "movq   %%mm4, %%mm6\n\t"
2665                 "movq   16(%1, %0, 4), %%mm0\n\t"
2666                 "movq   %%mm0, %%mm3\n\t"
2667                 "punpcklbw %%mm5, %%mm4\n\t"
2668                 "punpcklbw %%mm4, %%mm0\n\t" /* Y U4 Y V4 Y U4 Y V4*/
2669                 "punpckhbw %%mm4, %%mm3\n\t" /* Y U5 Y V5 Y U5 Y V5*/
2670                 MOVNTQ" %%mm0, 32(%4, %0, 8)\n\t"
2671                 MOVNTQ" %%mm3, 40(%4, %0, 8)\n\t"
2672                 
2673                 "punpckhbw %%mm5, %%mm6\n\t"
2674                 "movq   24(%1, %0, 4), %%mm0\n\t"
2675                 "movq   %%mm0, %%mm3\n\t"
2676                 "punpcklbw %%mm6, %%mm0\n\t" /* Y U6 Y V6 Y U6 Y V6*/
2677                 "punpckhbw %%mm6, %%mm3\n\t" /* Y U7 Y V7 Y U7 Y V7*/
2678                 MOVNTQ" %%mm0, 48(%4, %0, 8)\n\t"
2679                 MOVNTQ" %%mm3, 56(%4, %0, 8)\n\t"
2680
2681                 : "+r" (x)
2682                 : "r"(yp), "r" (up), "r"(vp), "r"(d)
2683                 :"memory");
2684         }
2685 #endif
2686         for(; x<w; x++)
2687         {
2688             const long x2= x<<2;
2689             d[8*x+0]=yp[x2];
2690             d[8*x+1]=up[x];
2691             d[8*x+2]=yp[x2+1];
2692             d[8*x+3]=vp[x];
2693             d[8*x+4]=yp[x2+2];
2694             d[8*x+5]=up[x];
2695             d[8*x+6]=yp[x2+3];
2696             d[8*x+7]=vp[x];
2697         }
2698     }
2699 #ifdef HAVE_MMX
2700         asm(
2701                 EMMS" \n\t"
2702                 SFENCE" \n\t"
2703                 ::: "memory"
2704                 );
2705 #endif
2706 }
2707
2708 static inline void RENAME(rgb2rgb_init)(void){
2709         rgb15to16= RENAME(rgb15to16);
2710         rgb15to24= RENAME(rgb15to24);
2711         rgb15to32= RENAME(rgb15to32);
2712         rgb16to24= RENAME(rgb16to24);
2713         rgb16to32= RENAME(rgb16to32);
2714         rgb16to15= RENAME(rgb16to15);
2715         rgb24to16= RENAME(rgb24to16);
2716         rgb24to15= RENAME(rgb24to15);
2717         rgb24to32= RENAME(rgb24to32);
2718         rgb32to16= RENAME(rgb32to16);
2719         rgb32to15= RENAME(rgb32to15);
2720         rgb32to24= RENAME(rgb32to24);
2721         rgb24tobgr15= RENAME(rgb24tobgr15);
2722         rgb24tobgr16= RENAME(rgb24tobgr16);
2723         rgb24tobgr24= RENAME(rgb24tobgr24);
2724         rgb32tobgr32= RENAME(rgb32tobgr32);
2725         rgb32tobgr16= RENAME(rgb32tobgr16);
2726         rgb32tobgr15= RENAME(rgb32tobgr15);
2727         yv12toyuy2= RENAME(yv12toyuy2);
2728         yv12touyvy= RENAME(yv12touyvy);
2729         yuv422ptoyuy2= RENAME(yuv422ptoyuy2);
2730         yuy2toyv12= RENAME(yuy2toyv12);
2731 //      uyvytoyv12= RENAME(uyvytoyv12);
2732 //      yvu9toyv12= RENAME(yvu9toyv12);
2733         planar2x= RENAME(planar2x);
2734         rgb24toyv12= RENAME(rgb24toyv12);
2735         interleaveBytes= RENAME(interleaveBytes);
2736         vu9_to_vu12= RENAME(vu9_to_vu12);
2737         yvu9_to_yuy2= RENAME(yvu9_to_yuy2);
2738 }