b08778a3a385d9b7e2a353f10d9673e2ce82281e
[ffmpeg.git] / postproc / rgb2rgb_template.c
1 /*
2  *
3  *  rgb2rgb.c, Software RGB to RGB convertor
4  *  pluralize by Software PAL8 to RGB convertor
5  *               Software YUV to YUV convertor
6  *               Software YUV to RGB convertor
7  *  Written by Nick Kurshev.
8  *  palette & yuv & runtime cpu stuff by Michael (michaelni@gmx.at) (under GPL)
9  *  lot of big-endian byteorder fixes by Alex Beregszaszi
10  */
11
12 #include <stddef.h>
13 #include <inttypes.h> /* for __WORDSIZE */
14
15 #ifndef __WORDSIZE
16 // #warning You have misconfigured system and probably will lose performance!
17 #define __WORDSIZE MP_WORDSIZE
18 #endif
19
20 #undef PREFETCH
21 #undef MOVNTQ
22 #undef EMMS
23 #undef SFENCE
24 #undef MMREG_SIZE
25 #undef PREFETCHW
26 #undef PAVGB
27
28 #ifdef HAVE_SSE2
29 #define MMREG_SIZE 16
30 #else
31 #define MMREG_SIZE 8
32 #endif
33
34 #ifdef HAVE_3DNOW
35 #define PREFETCH  "prefetch"
36 #define PREFETCHW "prefetchw"
37 #define PAVGB     "pavgusb"
38 #elif defined ( HAVE_MMX2 )
39 #define PREFETCH "prefetchnta"
40 #define PREFETCHW "prefetcht0"
41 #define PAVGB     "pavgb"
42 #else
43 #define PREFETCH "/nop"
44 #define PREFETCHW "/nop"
45 #endif
46
47 #ifdef HAVE_3DNOW
48 /* On K6 femms is faster of emms. On K7 femms is directly mapped on emms. */
49 #define EMMS     "femms"
50 #else
51 #define EMMS     "emms"
52 #endif
53
54 #ifdef HAVE_MMX2
55 #define MOVNTQ "movntq"
56 #define SFENCE "sfence"
57 #else
58 #define MOVNTQ "movq"
59 #define SFENCE "/nop"
60 #endif
61
62 static inline void RENAME(rgb24to32)(const uint8_t *src,uint8_t *dst,long src_size)
63 {
64   uint8_t *dest = dst;
65   const uint8_t *s = src;
66   const uint8_t *end;
67 #ifdef HAVE_MMX
68   const uint8_t *mm_end;
69 #endif
70   end = s + src_size;
71 #ifdef HAVE_MMX
72   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
73   mm_end = end - 23;
74   __asm __volatile("movq        %0, %%mm7"::"m"(mask32):"memory");
75   while(s < mm_end)
76   {
77     __asm __volatile(
78         PREFETCH"       32%1\n\t"
79         "movd   %1, %%mm0\n\t"
80         "punpckldq 3%1, %%mm0\n\t"
81         "movd   6%1, %%mm1\n\t"
82         "punpckldq 9%1, %%mm1\n\t"
83         "movd   12%1, %%mm2\n\t"
84         "punpckldq 15%1, %%mm2\n\t"
85         "movd   18%1, %%mm3\n\t"
86         "punpckldq 21%1, %%mm3\n\t"
87         "pand   %%mm7, %%mm0\n\t"
88         "pand   %%mm7, %%mm1\n\t"
89         "pand   %%mm7, %%mm2\n\t"
90         "pand   %%mm7, %%mm3\n\t"
91         MOVNTQ" %%mm0, %0\n\t"
92         MOVNTQ" %%mm1, 8%0\n\t"
93         MOVNTQ" %%mm2, 16%0\n\t"
94         MOVNTQ" %%mm3, 24%0"
95         :"=m"(*dest)
96         :"m"(*s)
97         :"memory");
98     dest += 32;
99     s += 24;
100   }
101   __asm __volatile(SFENCE:::"memory");
102   __asm __volatile(EMMS:::"memory");
103 #endif
104   while(s < end)
105   {
106 #ifdef WORDS_BIGENDIAN
107     /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */
108     *dest++ = 0;
109     *dest++ = s[2];
110     *dest++ = s[1];
111     *dest++ = s[0];
112     s+=3;
113 #else
114     *dest++ = *s++;
115     *dest++ = *s++;
116     *dest++ = *s++;
117     *dest++ = 0;
118 #endif
119   }
120 }
121
122 static inline void RENAME(rgb32to24)(const uint8_t *src,uint8_t *dst,long src_size)
123 {
124   uint8_t *dest = dst;
125   const uint8_t *s = src;
126   const uint8_t *end;
127 #ifdef HAVE_MMX
128   const uint8_t *mm_end;
129 #endif
130   end = s + src_size;
131 #ifdef HAVE_MMX
132   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
133   mm_end = end - 31;
134   while(s < mm_end)
135   {
136     __asm __volatile(
137         PREFETCH"       32%1\n\t"
138         "movq   %1, %%mm0\n\t"
139         "movq   8%1, %%mm1\n\t"
140         "movq   16%1, %%mm4\n\t"
141         "movq   24%1, %%mm5\n\t"
142         "movq   %%mm0, %%mm2\n\t"
143         "movq   %%mm1, %%mm3\n\t"
144         "movq   %%mm4, %%mm6\n\t"
145         "movq   %%mm5, %%mm7\n\t"
146         "psrlq  $8, %%mm2\n\t"
147         "psrlq  $8, %%mm3\n\t"
148         "psrlq  $8, %%mm6\n\t"
149         "psrlq  $8, %%mm7\n\t"
150         "pand   %2, %%mm0\n\t"
151         "pand   %2, %%mm1\n\t"
152         "pand   %2, %%mm4\n\t"
153         "pand   %2, %%mm5\n\t"
154         "pand   %3, %%mm2\n\t"
155         "pand   %3, %%mm3\n\t"
156         "pand   %3, %%mm6\n\t"
157         "pand   %3, %%mm7\n\t"
158         "por    %%mm2, %%mm0\n\t"
159         "por    %%mm3, %%mm1\n\t"
160         "por    %%mm6, %%mm4\n\t"
161         "por    %%mm7, %%mm5\n\t"
162
163         "movq   %%mm1, %%mm2\n\t"
164         "movq   %%mm4, %%mm3\n\t"
165         "psllq  $48, %%mm2\n\t"
166         "psllq  $32, %%mm3\n\t"
167         "pand   %4, %%mm2\n\t"
168         "pand   %5, %%mm3\n\t"
169         "por    %%mm2, %%mm0\n\t"
170         "psrlq  $16, %%mm1\n\t"
171         "psrlq  $32, %%mm4\n\t"
172         "psllq  $16, %%mm5\n\t"
173         "por    %%mm3, %%mm1\n\t"
174         "pand   %6, %%mm5\n\t"
175         "por    %%mm5, %%mm4\n\t"
176
177         MOVNTQ" %%mm0, %0\n\t"
178         MOVNTQ" %%mm1, 8%0\n\t"
179         MOVNTQ" %%mm4, 16%0"
180         :"=m"(*dest)
181         :"m"(*s),"m"(mask24l),
182          "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
183         :"memory");
184     dest += 24;
185     s += 32;
186   }
187   __asm __volatile(SFENCE:::"memory");
188   __asm __volatile(EMMS:::"memory");
189 #endif
190   while(s < end)
191   {
192 #ifdef WORDS_BIGENDIAN
193     /* RGB32 (= A,B,G,R) -> RGB24 (= R,G,B) */
194     s++;
195     dest[2] = *s++;
196     dest[1] = *s++;
197     dest[0] = *s++;
198     dest += 3;
199 #else
200     *dest++ = *s++;
201     *dest++ = *s++;
202     *dest++ = *s++;
203     s++;
204 #endif
205   }
206 }
207
208 /*
209  Original by Strepto/Astral
210  ported to gcc & bugfixed : A'rpi
211  MMX2, 3DNOW optimization by Nick Kurshev
212  32bit c version, and and&add trick by Michael Niedermayer
213 */
214 static inline void RENAME(rgb15to16)(const uint8_t *src,uint8_t *dst,long src_size)
215 {
216   register const uint8_t* s=src;
217   register uint8_t* d=dst;
218   register const uint8_t *end;
219   const uint8_t *mm_end;
220   end = s + src_size;
221 #ifdef HAVE_MMX
222   __asm __volatile(PREFETCH"    %0"::"m"(*s));
223   __asm __volatile("movq        %0, %%mm4"::"m"(mask15s));
224   mm_end = end - 15;
225   while(s<mm_end)
226   {
227         __asm __volatile(
228                 PREFETCH"       32%1\n\t"
229                 "movq   %1, %%mm0\n\t"
230                 "movq   8%1, %%mm2\n\t"
231                 "movq   %%mm0, %%mm1\n\t"
232                 "movq   %%mm2, %%mm3\n\t"
233                 "pand   %%mm4, %%mm0\n\t"
234                 "pand   %%mm4, %%mm2\n\t"
235                 "paddw  %%mm1, %%mm0\n\t"
236                 "paddw  %%mm3, %%mm2\n\t"
237                 MOVNTQ" %%mm0, %0\n\t"
238                 MOVNTQ" %%mm2, 8%0"
239                 :"=m"(*d)
240                 :"m"(*s)
241                 );
242         d+=16;
243         s+=16;
244   }
245   __asm __volatile(SFENCE:::"memory");
246   __asm __volatile(EMMS:::"memory");
247 #endif
248     mm_end = end - 3;
249     while(s < mm_end)
250     {
251         register unsigned x= *((uint32_t *)s);
252         *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
253         d+=4;
254         s+=4;
255     }
256     if(s < end)
257     {
258         register unsigned short x= *((uint16_t *)s);
259         *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
260     }
261 }
262
263 static inline void RENAME(rgb16to15)(const uint8_t *src,uint8_t *dst,long src_size)
264 {
265   register const uint8_t* s=src;
266   register uint8_t* d=dst;
267   register const uint8_t *end;
268   const uint8_t *mm_end;
269   end = s + src_size;
270 #ifdef HAVE_MMX
271   __asm __volatile(PREFETCH"    %0"::"m"(*s));
272   __asm __volatile("movq        %0, %%mm7"::"m"(mask15rg));
273   __asm __volatile("movq        %0, %%mm6"::"m"(mask15b));
274   mm_end = end - 15;
275   while(s<mm_end)
276   {
277         __asm __volatile(
278                 PREFETCH"       32%1\n\t"
279                 "movq   %1, %%mm0\n\t"
280                 "movq   8%1, %%mm2\n\t"
281                 "movq   %%mm0, %%mm1\n\t"
282                 "movq   %%mm2, %%mm3\n\t"
283                 "psrlq  $1, %%mm0\n\t"
284                 "psrlq  $1, %%mm2\n\t"
285                 "pand   %%mm7, %%mm0\n\t"
286                 "pand   %%mm7, %%mm2\n\t"
287                 "pand   %%mm6, %%mm1\n\t"
288                 "pand   %%mm6, %%mm3\n\t"
289                 "por    %%mm1, %%mm0\n\t"
290                 "por    %%mm3, %%mm2\n\t"
291                 MOVNTQ" %%mm0, %0\n\t"
292                 MOVNTQ" %%mm2, 8%0"
293                 :"=m"(*d)
294                 :"m"(*s)
295                 );
296         d+=16;
297         s+=16;
298   }
299   __asm __volatile(SFENCE:::"memory");
300   __asm __volatile(EMMS:::"memory");
301 #endif
302     mm_end = end - 3;
303     while(s < mm_end)
304     {
305         register uint32_t x= *((uint32_t *)s);
306         *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
307         s+=4;
308         d+=4;
309     }
310     if(s < end)
311     {
312         register uint16_t x= *((uint16_t *)s);
313         *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
314         s+=2;
315         d+=2;
316     }
317 }
318
319 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, long src_size)
320 {
321         const uint8_t *s = src;
322         const uint8_t *end;
323 #ifdef HAVE_MMX
324         const uint8_t *mm_end;
325 #endif
326         uint16_t *d = (uint16_t *)dst;
327         end = s + src_size;
328 #ifdef HAVE_MMX
329         mm_end = end - 15;
330 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
331         asm volatile(
332                 "movq %3, %%mm5                 \n\t"
333                 "movq %4, %%mm6                 \n\t"
334                 "movq %5, %%mm7                 \n\t"
335                 ".balign 16                     \n\t"
336                 "1:                             \n\t"
337                 PREFETCH" 32(%1)                \n\t"
338                 "movd   (%1), %%mm0             \n\t"
339                 "movd   4(%1), %%mm3            \n\t"
340                 "punpckldq 8(%1), %%mm0         \n\t"
341                 "punpckldq 12(%1), %%mm3        \n\t"
342                 "movq %%mm0, %%mm1              \n\t"
343                 "movq %%mm3, %%mm4              \n\t"
344                 "pand %%mm6, %%mm0              \n\t"
345                 "pand %%mm6, %%mm3              \n\t"
346                 "pmaddwd %%mm7, %%mm0           \n\t"
347                 "pmaddwd %%mm7, %%mm3           \n\t"
348                 "pand %%mm5, %%mm1              \n\t"
349                 "pand %%mm5, %%mm4              \n\t"
350                 "por %%mm1, %%mm0               \n\t"   
351                 "por %%mm4, %%mm3               \n\t"
352                 "psrld $5, %%mm0                \n\t"
353                 "pslld $11, %%mm3               \n\t"
354                 "por %%mm3, %%mm0               \n\t"
355                 MOVNTQ" %%mm0, (%0)             \n\t"
356                 "add $16, %1                    \n\t"
357                 "add $8, %0                     \n\t"
358                 "cmp %2, %1                     \n\t"
359                 " jb 1b                         \n\t"
360                 : "+r" (d), "+r"(s)
361                 : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
362         );
363 #else
364         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
365         __asm __volatile(
366             "movq       %0, %%mm7\n\t"
367             "movq       %1, %%mm6\n\t"
368             ::"m"(red_16mask),"m"(green_16mask));
369         while(s < mm_end)
370         {
371             __asm __volatile(
372                 PREFETCH" 32%1\n\t"
373                 "movd   %1, %%mm0\n\t"
374                 "movd   4%1, %%mm3\n\t"
375                 "punpckldq 8%1, %%mm0\n\t"
376                 "punpckldq 12%1, %%mm3\n\t"
377                 "movq   %%mm0, %%mm1\n\t"
378                 "movq   %%mm0, %%mm2\n\t"
379                 "movq   %%mm3, %%mm4\n\t"
380                 "movq   %%mm3, %%mm5\n\t"
381                 "psrlq  $3, %%mm0\n\t"
382                 "psrlq  $3, %%mm3\n\t"
383                 "pand   %2, %%mm0\n\t"
384                 "pand   %2, %%mm3\n\t"
385                 "psrlq  $5, %%mm1\n\t"
386                 "psrlq  $5, %%mm4\n\t"
387                 "pand   %%mm6, %%mm1\n\t"
388                 "pand   %%mm6, %%mm4\n\t"
389                 "psrlq  $8, %%mm2\n\t"
390                 "psrlq  $8, %%mm5\n\t"
391                 "pand   %%mm7, %%mm2\n\t"
392                 "pand   %%mm7, %%mm5\n\t"
393                 "por    %%mm1, %%mm0\n\t"
394                 "por    %%mm4, %%mm3\n\t"
395                 "por    %%mm2, %%mm0\n\t"
396                 "por    %%mm5, %%mm3\n\t"
397                 "psllq  $16, %%mm3\n\t"
398                 "por    %%mm3, %%mm0\n\t"
399                 MOVNTQ" %%mm0, %0\n\t"
400                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
401                 d += 4;
402                 s += 16;
403         }
404 #endif
405         __asm __volatile(SFENCE:::"memory");
406         __asm __volatile(EMMS:::"memory");
407 #endif
408         while(s < end)
409         {
410                 register int rgb = *(uint32_t*)s; s += 4;
411                 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
412         }
413 }
414
415 static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
416 {
417         const uint8_t *s = src;
418         const uint8_t *end;
419 #ifdef HAVE_MMX
420         const uint8_t *mm_end;
421 #endif
422         uint16_t *d = (uint16_t *)dst;
423         end = s + src_size;
424 #ifdef HAVE_MMX
425         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
426         __asm __volatile(
427             "movq       %0, %%mm7\n\t"
428             "movq       %1, %%mm6\n\t"
429             ::"m"(red_16mask),"m"(green_16mask));
430         mm_end = end - 15;
431         while(s < mm_end)
432         {
433             __asm __volatile(
434                 PREFETCH" 32%1\n\t"
435                 "movd   %1, %%mm0\n\t"
436                 "movd   4%1, %%mm3\n\t"
437                 "punpckldq 8%1, %%mm0\n\t"
438                 "punpckldq 12%1, %%mm3\n\t"
439                 "movq   %%mm0, %%mm1\n\t"
440                 "movq   %%mm0, %%mm2\n\t"
441                 "movq   %%mm3, %%mm4\n\t"
442                 "movq   %%mm3, %%mm5\n\t"
443                 "psllq  $8, %%mm0\n\t"
444                 "psllq  $8, %%mm3\n\t"
445                 "pand   %%mm7, %%mm0\n\t"
446                 "pand   %%mm7, %%mm3\n\t"
447                 "psrlq  $5, %%mm1\n\t"
448                 "psrlq  $5, %%mm4\n\t"
449                 "pand   %%mm6, %%mm1\n\t"
450                 "pand   %%mm6, %%mm4\n\t"
451                 "psrlq  $19, %%mm2\n\t"
452                 "psrlq  $19, %%mm5\n\t"
453                 "pand   %2, %%mm2\n\t"
454                 "pand   %2, %%mm5\n\t"
455                 "por    %%mm1, %%mm0\n\t"
456                 "por    %%mm4, %%mm3\n\t"
457                 "por    %%mm2, %%mm0\n\t"
458                 "por    %%mm5, %%mm3\n\t"
459                 "psllq  $16, %%mm3\n\t"
460                 "por    %%mm3, %%mm0\n\t"
461                 MOVNTQ" %%mm0, %0\n\t"
462                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
463                 d += 4;
464                 s += 16;
465         }
466         __asm __volatile(SFENCE:::"memory");
467         __asm __volatile(EMMS:::"memory");
468 #endif
469         while(s < end)
470         {
471                 // FIXME on bigendian
472                 /* Looks bigendian-OK to me. --Pac. */
473                 const int src= *s; s += 4;
474                 *d++ = ((src&0xF8)<<8) + ((src&0xFC00)>>5) + ((src&0xF80000)>>19);
475         }
476 }
477
478 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, long src_size)
479 {
480         const uint8_t *s = src;
481         const uint8_t *end;
482 #ifdef HAVE_MMX
483         const uint8_t *mm_end;
484 #endif
485         uint16_t *d = (uint16_t *)dst;
486         end = s + src_size;
487 #ifdef HAVE_MMX
488         mm_end = end - 15;
489 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
490         asm volatile(
491                 "movq %3, %%mm5                 \n\t"
492                 "movq %4, %%mm6                 \n\t"
493                 "movq %5, %%mm7                 \n\t"
494                 ".balign 16                     \n\t"
495                 "1:                             \n\t"
496                 PREFETCH" 32(%1)                \n\t"
497                 "movd   (%1), %%mm0             \n\t"
498                 "movd   4(%1), %%mm3            \n\t"
499                 "punpckldq 8(%1), %%mm0         \n\t"
500                 "punpckldq 12(%1), %%mm3        \n\t"
501                 "movq %%mm0, %%mm1              \n\t"
502                 "movq %%mm3, %%mm4              \n\t"
503                 "pand %%mm6, %%mm0              \n\t"
504                 "pand %%mm6, %%mm3              \n\t"
505                 "pmaddwd %%mm7, %%mm0           \n\t"
506                 "pmaddwd %%mm7, %%mm3           \n\t"
507                 "pand %%mm5, %%mm1              \n\t"
508                 "pand %%mm5, %%mm4              \n\t"
509                 "por %%mm1, %%mm0               \n\t"   
510                 "por %%mm4, %%mm3               \n\t"
511                 "psrld $6, %%mm0                \n\t"
512                 "pslld $10, %%mm3               \n\t"
513                 "por %%mm3, %%mm0               \n\t"
514                 MOVNTQ" %%mm0, (%0)             \n\t"
515                 "add $16, %1                    \n\t"
516                 "add $8, %0                     \n\t"
517                 "cmp %2, %1                     \n\t"
518                 " jb 1b                         \n\t"
519                 : "+r" (d), "+r"(s)
520                 : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
521         );
522 #else
523         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
524         __asm __volatile(
525             "movq       %0, %%mm7\n\t"
526             "movq       %1, %%mm6\n\t"
527             ::"m"(red_15mask),"m"(green_15mask));
528         while(s < mm_end)
529         {
530             __asm __volatile(
531                 PREFETCH" 32%1\n\t"
532                 "movd   %1, %%mm0\n\t"
533                 "movd   4%1, %%mm3\n\t"
534                 "punpckldq 8%1, %%mm0\n\t"
535                 "punpckldq 12%1, %%mm3\n\t"
536                 "movq   %%mm0, %%mm1\n\t"
537                 "movq   %%mm0, %%mm2\n\t"
538                 "movq   %%mm3, %%mm4\n\t"
539                 "movq   %%mm3, %%mm5\n\t"
540                 "psrlq  $3, %%mm0\n\t"
541                 "psrlq  $3, %%mm3\n\t"
542                 "pand   %2, %%mm0\n\t"
543                 "pand   %2, %%mm3\n\t"
544                 "psrlq  $6, %%mm1\n\t"
545                 "psrlq  $6, %%mm4\n\t"
546                 "pand   %%mm6, %%mm1\n\t"
547                 "pand   %%mm6, %%mm4\n\t"
548                 "psrlq  $9, %%mm2\n\t"
549                 "psrlq  $9, %%mm5\n\t"
550                 "pand   %%mm7, %%mm2\n\t"
551                 "pand   %%mm7, %%mm5\n\t"
552                 "por    %%mm1, %%mm0\n\t"
553                 "por    %%mm4, %%mm3\n\t"
554                 "por    %%mm2, %%mm0\n\t"
555                 "por    %%mm5, %%mm3\n\t"
556                 "psllq  $16, %%mm3\n\t"
557                 "por    %%mm3, %%mm0\n\t"
558                 MOVNTQ" %%mm0, %0\n\t"
559                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
560                 d += 4;
561                 s += 16;
562         }
563 #endif
564         __asm __volatile(SFENCE:::"memory");
565         __asm __volatile(EMMS:::"memory");
566 #endif
567         while(s < end)
568         {
569                 // FIXME on bigendian
570                 /* Looks bigendian-OK to me. --Pac. */
571                 const int src= *s; s += 4;
572                 *d++ = ((src&0xFF)>>3) + ((src&0xF800)>>6) + ((src&0xF80000)>>9);
573         }
574 }
575
576 static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
577 {
578         const uint8_t *s = src;
579         const uint8_t *end;
580 #ifdef HAVE_MMX
581         const uint8_t *mm_end;
582 #endif
583         uint16_t *d = (uint16_t *)dst;
584         end = s + src_size;
585 #ifdef HAVE_MMX
586         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
587         __asm __volatile(
588             "movq       %0, %%mm7\n\t"
589             "movq       %1, %%mm6\n\t"
590             ::"m"(red_15mask),"m"(green_15mask));
591         mm_end = end - 15;
592         while(s < mm_end)
593         {
594             __asm __volatile(
595                 PREFETCH" 32%1\n\t"
596                 "movd   %1, %%mm0\n\t"
597                 "movd   4%1, %%mm3\n\t"
598                 "punpckldq 8%1, %%mm0\n\t"
599                 "punpckldq 12%1, %%mm3\n\t"
600                 "movq   %%mm0, %%mm1\n\t"
601                 "movq   %%mm0, %%mm2\n\t"
602                 "movq   %%mm3, %%mm4\n\t"
603                 "movq   %%mm3, %%mm5\n\t"
604                 "psllq  $7, %%mm0\n\t"
605                 "psllq  $7, %%mm3\n\t"
606                 "pand   %%mm7, %%mm0\n\t"
607                 "pand   %%mm7, %%mm3\n\t"
608                 "psrlq  $6, %%mm1\n\t"
609                 "psrlq  $6, %%mm4\n\t"
610                 "pand   %%mm6, %%mm1\n\t"
611                 "pand   %%mm6, %%mm4\n\t"
612                 "psrlq  $19, %%mm2\n\t"
613                 "psrlq  $19, %%mm5\n\t"
614                 "pand   %2, %%mm2\n\t"
615                 "pand   %2, %%mm5\n\t"
616                 "por    %%mm1, %%mm0\n\t"
617                 "por    %%mm4, %%mm3\n\t"
618                 "por    %%mm2, %%mm0\n\t"
619                 "por    %%mm5, %%mm3\n\t"
620                 "psllq  $16, %%mm3\n\t"
621                 "por    %%mm3, %%mm0\n\t"
622                 MOVNTQ" %%mm0, %0\n\t"
623                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
624                 d += 4;
625                 s += 16;
626         }
627         __asm __volatile(SFENCE:::"memory");
628         __asm __volatile(EMMS:::"memory");
629 #endif
630         while(s < end)
631         {
632                 // FIXME on bigendian
633                 /* Looks bigendian-OK to me. --Pac. */
634                 const int src= *s; s += 4;
635                 *d++ = ((src&0xF8)<<7) + ((src&0xF800)>>6) + ((src&0xF80000)>>19);
636         }
637 }
638
639 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, long src_size)
640 {
641         const uint8_t *s = src;
642         const uint8_t *end;
643 #ifdef HAVE_MMX
644         const uint8_t *mm_end;
645 #endif
646         uint16_t *d = (uint16_t *)dst;
647         end = s + src_size;
648 #ifdef HAVE_MMX
649         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
650         __asm __volatile(
651             "movq       %0, %%mm7\n\t"
652             "movq       %1, %%mm6\n\t"
653             ::"m"(red_16mask),"m"(green_16mask));
654         mm_end = end - 11;
655         while(s < mm_end)
656         {
657             __asm __volatile(
658                 PREFETCH" 32%1\n\t"
659                 "movd   %1, %%mm0\n\t"
660                 "movd   3%1, %%mm3\n\t"
661                 "punpckldq 6%1, %%mm0\n\t"
662                 "punpckldq 9%1, %%mm3\n\t"
663                 "movq   %%mm0, %%mm1\n\t"
664                 "movq   %%mm0, %%mm2\n\t"
665                 "movq   %%mm3, %%mm4\n\t"
666                 "movq   %%mm3, %%mm5\n\t"
667                 "psrlq  $3, %%mm0\n\t"
668                 "psrlq  $3, %%mm3\n\t"
669                 "pand   %2, %%mm0\n\t"
670                 "pand   %2, %%mm3\n\t"
671                 "psrlq  $5, %%mm1\n\t"
672                 "psrlq  $5, %%mm4\n\t"
673                 "pand   %%mm6, %%mm1\n\t"
674                 "pand   %%mm6, %%mm4\n\t"
675                 "psrlq  $8, %%mm2\n\t"
676                 "psrlq  $8, %%mm5\n\t"
677                 "pand   %%mm7, %%mm2\n\t"
678                 "pand   %%mm7, %%mm5\n\t"
679                 "por    %%mm1, %%mm0\n\t"
680                 "por    %%mm4, %%mm3\n\t"
681                 "por    %%mm2, %%mm0\n\t"
682                 "por    %%mm5, %%mm3\n\t"
683                 "psllq  $16, %%mm3\n\t"
684                 "por    %%mm3, %%mm0\n\t"
685                 MOVNTQ" %%mm0, %0\n\t"
686                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
687                 d += 4;
688                 s += 12;
689         }
690         __asm __volatile(SFENCE:::"memory");
691         __asm __volatile(EMMS:::"memory");
692 #endif
693         while(s < end)
694         {
695                 const int b= *s++;
696                 const int g= *s++;
697                 const int r= *s++;
698                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
699         }
700 }
701
702 static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
703 {
704         const uint8_t *s = src;
705         const uint8_t *end;
706 #ifdef HAVE_MMX
707         const uint8_t *mm_end;
708 #endif
709         uint16_t *d = (uint16_t *)dst;
710         end = s + src_size;
711 #ifdef HAVE_MMX
712         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
713         __asm __volatile(
714             "movq       %0, %%mm7\n\t"
715             "movq       %1, %%mm6\n\t"
716             ::"m"(red_16mask),"m"(green_16mask));
717         mm_end = end - 15;
718         while(s < mm_end)
719         {
720             __asm __volatile(
721                 PREFETCH" 32%1\n\t"
722                 "movd   %1, %%mm0\n\t"
723                 "movd   3%1, %%mm3\n\t"
724                 "punpckldq 6%1, %%mm0\n\t"
725                 "punpckldq 9%1, %%mm3\n\t"
726                 "movq   %%mm0, %%mm1\n\t"
727                 "movq   %%mm0, %%mm2\n\t"
728                 "movq   %%mm3, %%mm4\n\t"
729                 "movq   %%mm3, %%mm5\n\t"
730                 "psllq  $8, %%mm0\n\t"
731                 "psllq  $8, %%mm3\n\t"
732                 "pand   %%mm7, %%mm0\n\t"
733                 "pand   %%mm7, %%mm3\n\t"
734                 "psrlq  $5, %%mm1\n\t"
735                 "psrlq  $5, %%mm4\n\t"
736                 "pand   %%mm6, %%mm1\n\t"
737                 "pand   %%mm6, %%mm4\n\t"
738                 "psrlq  $19, %%mm2\n\t"
739                 "psrlq  $19, %%mm5\n\t"
740                 "pand   %2, %%mm2\n\t"
741                 "pand   %2, %%mm5\n\t"
742                 "por    %%mm1, %%mm0\n\t"
743                 "por    %%mm4, %%mm3\n\t"
744                 "por    %%mm2, %%mm0\n\t"
745                 "por    %%mm5, %%mm3\n\t"
746                 "psllq  $16, %%mm3\n\t"
747                 "por    %%mm3, %%mm0\n\t"
748                 MOVNTQ" %%mm0, %0\n\t"
749                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
750                 d += 4;
751                 s += 12;
752         }
753         __asm __volatile(SFENCE:::"memory");
754         __asm __volatile(EMMS:::"memory");
755 #endif
756         while(s < end)
757         {
758                 const int r= *s++;
759                 const int g= *s++;
760                 const int b= *s++;
761                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
762         }
763 }
764
765 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, long src_size)
766 {
767         const uint8_t *s = src;
768         const uint8_t *end;
769 #ifdef HAVE_MMX
770         const uint8_t *mm_end;
771 #endif
772         uint16_t *d = (uint16_t *)dst;
773         end = s + src_size;
774 #ifdef HAVE_MMX
775         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
776         __asm __volatile(
777             "movq       %0, %%mm7\n\t"
778             "movq       %1, %%mm6\n\t"
779             ::"m"(red_15mask),"m"(green_15mask));
780         mm_end = end - 11;
781         while(s < mm_end)
782         {
783             __asm __volatile(
784                 PREFETCH" 32%1\n\t"
785                 "movd   %1, %%mm0\n\t"
786                 "movd   3%1, %%mm3\n\t"
787                 "punpckldq 6%1, %%mm0\n\t"
788                 "punpckldq 9%1, %%mm3\n\t"
789                 "movq   %%mm0, %%mm1\n\t"
790                 "movq   %%mm0, %%mm2\n\t"
791                 "movq   %%mm3, %%mm4\n\t"
792                 "movq   %%mm3, %%mm5\n\t"
793                 "psrlq  $3, %%mm0\n\t"
794                 "psrlq  $3, %%mm3\n\t"
795                 "pand   %2, %%mm0\n\t"
796                 "pand   %2, %%mm3\n\t"
797                 "psrlq  $6, %%mm1\n\t"
798                 "psrlq  $6, %%mm4\n\t"
799                 "pand   %%mm6, %%mm1\n\t"
800                 "pand   %%mm6, %%mm4\n\t"
801                 "psrlq  $9, %%mm2\n\t"
802                 "psrlq  $9, %%mm5\n\t"
803                 "pand   %%mm7, %%mm2\n\t"
804                 "pand   %%mm7, %%mm5\n\t"
805                 "por    %%mm1, %%mm0\n\t"
806                 "por    %%mm4, %%mm3\n\t"
807                 "por    %%mm2, %%mm0\n\t"
808                 "por    %%mm5, %%mm3\n\t"
809                 "psllq  $16, %%mm3\n\t"
810                 "por    %%mm3, %%mm0\n\t"
811                 MOVNTQ" %%mm0, %0\n\t"
812                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
813                 d += 4;
814                 s += 12;
815         }
816         __asm __volatile(SFENCE:::"memory");
817         __asm __volatile(EMMS:::"memory");
818 #endif
819         while(s < end)
820         {
821                 const int b= *s++;
822                 const int g= *s++;
823                 const int r= *s++;
824                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
825         }
826 }
827
828 static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
829 {
830         const uint8_t *s = src;
831         const uint8_t *end;
832 #ifdef HAVE_MMX
833         const uint8_t *mm_end;
834 #endif
835         uint16_t *d = (uint16_t *)dst;
836         end = s + src_size;
837 #ifdef HAVE_MMX
838         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
839         __asm __volatile(
840             "movq       %0, %%mm7\n\t"
841             "movq       %1, %%mm6\n\t"
842             ::"m"(red_15mask),"m"(green_15mask));
843         mm_end = end - 15;
844         while(s < mm_end)
845         {
846             __asm __volatile(
847                 PREFETCH" 32%1\n\t"
848                 "movd   %1, %%mm0\n\t"
849                 "movd   3%1, %%mm3\n\t"
850                 "punpckldq 6%1, %%mm0\n\t"
851                 "punpckldq 9%1, %%mm3\n\t"
852                 "movq   %%mm0, %%mm1\n\t"
853                 "movq   %%mm0, %%mm2\n\t"
854                 "movq   %%mm3, %%mm4\n\t"
855                 "movq   %%mm3, %%mm5\n\t"
856                 "psllq  $7, %%mm0\n\t"
857                 "psllq  $7, %%mm3\n\t"
858                 "pand   %%mm7, %%mm0\n\t"
859                 "pand   %%mm7, %%mm3\n\t"
860                 "psrlq  $6, %%mm1\n\t"
861                 "psrlq  $6, %%mm4\n\t"
862                 "pand   %%mm6, %%mm1\n\t"
863                 "pand   %%mm6, %%mm4\n\t"
864                 "psrlq  $19, %%mm2\n\t"
865                 "psrlq  $19, %%mm5\n\t"
866                 "pand   %2, %%mm2\n\t"
867                 "pand   %2, %%mm5\n\t"
868                 "por    %%mm1, %%mm0\n\t"
869                 "por    %%mm4, %%mm3\n\t"
870                 "por    %%mm2, %%mm0\n\t"
871                 "por    %%mm5, %%mm3\n\t"
872                 "psllq  $16, %%mm3\n\t"
873                 "por    %%mm3, %%mm0\n\t"
874                 MOVNTQ" %%mm0, %0\n\t"
875                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
876                 d += 4;
877                 s += 12;
878         }
879         __asm __volatile(SFENCE:::"memory");
880         __asm __volatile(EMMS:::"memory");
881 #endif
882         while(s < end)
883         {
884                 const int r= *s++;
885                 const int g= *s++;
886                 const int b= *s++;
887                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
888         }
889 }
890
891 /*
892   I use here less accurate approximation by simply
893  left-shifting the input
894   value and filling the low order bits with
895  zeroes. This method improves png's
896   compression but this scheme cannot reproduce white exactly, since it does not
897   generate an all-ones maximum value; the net effect is to darken the
898   image slightly.
899
900   The better method should be "left bit replication":
901
902    4 3 2 1 0
903    ---------
904    1 1 0 1 1
905
906    7 6 5 4 3  2 1 0
907    ----------------
908    1 1 0 1 1  1 1 0
909    |=======|  |===|
910        |      Leftmost Bits Repeated to Fill Open Bits
911        |
912    Original Bits
913 */
914 static inline void RENAME(rgb15to24)(const uint8_t *src, uint8_t *dst, long src_size)
915 {
916         const uint16_t *end;
917 #ifdef HAVE_MMX
918         const uint16_t *mm_end;
919 #endif
920         uint8_t *d = (uint8_t *)dst;
921         const uint16_t *s = (uint16_t *)src;
922         end = s + src_size/2;
923 #ifdef HAVE_MMX
924         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
925         mm_end = end - 7;
926         while(s < mm_end)
927         {
928             __asm __volatile(
929                 PREFETCH" 32%1\n\t"
930                 "movq   %1, %%mm0\n\t"
931                 "movq   %1, %%mm1\n\t"
932                 "movq   %1, %%mm2\n\t"
933                 "pand   %2, %%mm0\n\t"
934                 "pand   %3, %%mm1\n\t"
935                 "pand   %4, %%mm2\n\t"
936                 "psllq  $3, %%mm0\n\t"
937                 "psrlq  $2, %%mm1\n\t"
938                 "psrlq  $7, %%mm2\n\t"
939                 "movq   %%mm0, %%mm3\n\t"
940                 "movq   %%mm1, %%mm4\n\t"
941                 "movq   %%mm2, %%mm5\n\t"
942                 "punpcklwd %5, %%mm0\n\t"
943                 "punpcklwd %5, %%mm1\n\t"
944                 "punpcklwd %5, %%mm2\n\t"
945                 "punpckhwd %5, %%mm3\n\t"
946                 "punpckhwd %5, %%mm4\n\t"
947                 "punpckhwd %5, %%mm5\n\t"
948                 "psllq  $8, %%mm1\n\t"
949                 "psllq  $16, %%mm2\n\t"
950                 "por    %%mm1, %%mm0\n\t"
951                 "por    %%mm2, %%mm0\n\t"
952                 "psllq  $8, %%mm4\n\t"
953                 "psllq  $16, %%mm5\n\t"
954                 "por    %%mm4, %%mm3\n\t"
955                 "por    %%mm5, %%mm3\n\t"
956
957                 "movq   %%mm0, %%mm6\n\t"
958                 "movq   %%mm3, %%mm7\n\t"
959                 
960                 "movq   8%1, %%mm0\n\t"
961                 "movq   8%1, %%mm1\n\t"
962                 "movq   8%1, %%mm2\n\t"
963                 "pand   %2, %%mm0\n\t"
964                 "pand   %3, %%mm1\n\t"
965                 "pand   %4, %%mm2\n\t"
966                 "psllq  $3, %%mm0\n\t"
967                 "psrlq  $2, %%mm1\n\t"
968                 "psrlq  $7, %%mm2\n\t"
969                 "movq   %%mm0, %%mm3\n\t"
970                 "movq   %%mm1, %%mm4\n\t"
971                 "movq   %%mm2, %%mm5\n\t"
972                 "punpcklwd %5, %%mm0\n\t"
973                 "punpcklwd %5, %%mm1\n\t"
974                 "punpcklwd %5, %%mm2\n\t"
975                 "punpckhwd %5, %%mm3\n\t"
976                 "punpckhwd %5, %%mm4\n\t"
977                 "punpckhwd %5, %%mm5\n\t"
978                 "psllq  $8, %%mm1\n\t"
979                 "psllq  $16, %%mm2\n\t"
980                 "por    %%mm1, %%mm0\n\t"
981                 "por    %%mm2, %%mm0\n\t"
982                 "psllq  $8, %%mm4\n\t"
983                 "psllq  $16, %%mm5\n\t"
984                 "por    %%mm4, %%mm3\n\t"
985                 "por    %%mm5, %%mm3\n\t"
986
987                 :"=m"(*d)
988                 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)
989                 :"memory");
990             /* Borrowed 32 to 24 */
991             __asm __volatile(
992                 "movq   %%mm0, %%mm4\n\t"
993                 "movq   %%mm3, %%mm5\n\t"
994                 "movq   %%mm6, %%mm0\n\t"
995                 "movq   %%mm7, %%mm1\n\t"
996                 
997                 "movq   %%mm4, %%mm6\n\t"
998                 "movq   %%mm5, %%mm7\n\t"
999                 "movq   %%mm0, %%mm2\n\t"
1000                 "movq   %%mm1, %%mm3\n\t"
1001
1002                 "psrlq  $8, %%mm2\n\t"
1003                 "psrlq  $8, %%mm3\n\t"
1004                 "psrlq  $8, %%mm6\n\t"
1005                 "psrlq  $8, %%mm7\n\t"
1006                 "pand   %2, %%mm0\n\t"
1007                 "pand   %2, %%mm1\n\t"
1008                 "pand   %2, %%mm4\n\t"
1009                 "pand   %2, %%mm5\n\t"
1010                 "pand   %3, %%mm2\n\t"
1011                 "pand   %3, %%mm3\n\t"
1012                 "pand   %3, %%mm6\n\t"
1013                 "pand   %3, %%mm7\n\t"
1014                 "por    %%mm2, %%mm0\n\t"
1015                 "por    %%mm3, %%mm1\n\t"
1016                 "por    %%mm6, %%mm4\n\t"
1017                 "por    %%mm7, %%mm5\n\t"
1018
1019                 "movq   %%mm1, %%mm2\n\t"
1020                 "movq   %%mm4, %%mm3\n\t"
1021                 "psllq  $48, %%mm2\n\t"
1022                 "psllq  $32, %%mm3\n\t"
1023                 "pand   %4, %%mm2\n\t"
1024                 "pand   %5, %%mm3\n\t"
1025                 "por    %%mm2, %%mm0\n\t"
1026                 "psrlq  $16, %%mm1\n\t"
1027                 "psrlq  $32, %%mm4\n\t"
1028                 "psllq  $16, %%mm5\n\t"
1029                 "por    %%mm3, %%mm1\n\t"
1030                 "pand   %6, %%mm5\n\t"
1031                 "por    %%mm5, %%mm4\n\t"
1032
1033                 MOVNTQ" %%mm0, %0\n\t"
1034                 MOVNTQ" %%mm1, 8%0\n\t"
1035                 MOVNTQ" %%mm4, 16%0"
1036
1037                 :"=m"(*d)
1038                 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1039                 :"memory");
1040                 d += 24;
1041                 s += 8;
1042         }
1043         __asm __volatile(SFENCE:::"memory");
1044         __asm __volatile(EMMS:::"memory");
1045 #endif
1046         while(s < end)
1047         {
1048                 register uint16_t bgr;
1049                 bgr = *s++;
1050                 *d++ = (bgr&0x1F)<<3;
1051                 *d++ = (bgr&0x3E0)>>2;
1052                 *d++ = (bgr&0x7C00)>>7;
1053         }
1054 }
1055
1056 static inline void RENAME(rgb16to24)(const uint8_t *src, uint8_t *dst, long src_size)
1057 {
1058         const uint16_t *end;
1059 #ifdef HAVE_MMX
1060         const uint16_t *mm_end;
1061 #endif
1062         uint8_t *d = (uint8_t *)dst;
1063         const uint16_t *s = (const uint16_t *)src;
1064         end = s + src_size/2;
1065 #ifdef HAVE_MMX
1066         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1067         mm_end = end - 7;
1068         while(s < mm_end)
1069         {
1070             __asm __volatile(
1071                 PREFETCH" 32%1\n\t"
1072                 "movq   %1, %%mm0\n\t"
1073                 "movq   %1, %%mm1\n\t"
1074                 "movq   %1, %%mm2\n\t"
1075                 "pand   %2, %%mm0\n\t"
1076                 "pand   %3, %%mm1\n\t"
1077                 "pand   %4, %%mm2\n\t"
1078                 "psllq  $3, %%mm0\n\t"
1079                 "psrlq  $3, %%mm1\n\t"
1080                 "psrlq  $8, %%mm2\n\t"
1081                 "movq   %%mm0, %%mm3\n\t"
1082                 "movq   %%mm1, %%mm4\n\t"
1083                 "movq   %%mm2, %%mm5\n\t"
1084                 "punpcklwd %5, %%mm0\n\t"
1085                 "punpcklwd %5, %%mm1\n\t"
1086                 "punpcklwd %5, %%mm2\n\t"
1087                 "punpckhwd %5, %%mm3\n\t"
1088                 "punpckhwd %5, %%mm4\n\t"
1089                 "punpckhwd %5, %%mm5\n\t"
1090                 "psllq  $8, %%mm1\n\t"
1091                 "psllq  $16, %%mm2\n\t"
1092                 "por    %%mm1, %%mm0\n\t"
1093                 "por    %%mm2, %%mm0\n\t"
1094                 "psllq  $8, %%mm4\n\t"
1095                 "psllq  $16, %%mm5\n\t"
1096                 "por    %%mm4, %%mm3\n\t"
1097                 "por    %%mm5, %%mm3\n\t"
1098                 
1099                 "movq   %%mm0, %%mm6\n\t"
1100                 "movq   %%mm3, %%mm7\n\t"
1101
1102                 "movq   8%1, %%mm0\n\t"
1103                 "movq   8%1, %%mm1\n\t"
1104                 "movq   8%1, %%mm2\n\t"
1105                 "pand   %2, %%mm0\n\t"
1106                 "pand   %3, %%mm1\n\t"
1107                 "pand   %4, %%mm2\n\t"
1108                 "psllq  $3, %%mm0\n\t"
1109                 "psrlq  $3, %%mm1\n\t"
1110                 "psrlq  $8, %%mm2\n\t"
1111                 "movq   %%mm0, %%mm3\n\t"
1112                 "movq   %%mm1, %%mm4\n\t"
1113                 "movq   %%mm2, %%mm5\n\t"
1114                 "punpcklwd %5, %%mm0\n\t"
1115                 "punpcklwd %5, %%mm1\n\t"
1116                 "punpcklwd %5, %%mm2\n\t"
1117                 "punpckhwd %5, %%mm3\n\t"
1118                 "punpckhwd %5, %%mm4\n\t"
1119                 "punpckhwd %5, %%mm5\n\t"
1120                 "psllq  $8, %%mm1\n\t"
1121                 "psllq  $16, %%mm2\n\t"
1122                 "por    %%mm1, %%mm0\n\t"
1123                 "por    %%mm2, %%mm0\n\t"
1124                 "psllq  $8, %%mm4\n\t"
1125                 "psllq  $16, %%mm5\n\t"
1126                 "por    %%mm4, %%mm3\n\t"
1127                 "por    %%mm5, %%mm3\n\t"
1128                 :"=m"(*d)
1129                 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)           
1130                 :"memory");
1131             /* Borrowed 32 to 24 */
1132             __asm __volatile(
1133                 "movq   %%mm0, %%mm4\n\t"
1134                 "movq   %%mm3, %%mm5\n\t"
1135                 "movq   %%mm6, %%mm0\n\t"
1136                 "movq   %%mm7, %%mm1\n\t"
1137                 
1138                 "movq   %%mm4, %%mm6\n\t"
1139                 "movq   %%mm5, %%mm7\n\t"
1140                 "movq   %%mm0, %%mm2\n\t"
1141                 "movq   %%mm1, %%mm3\n\t"
1142
1143                 "psrlq  $8, %%mm2\n\t"
1144                 "psrlq  $8, %%mm3\n\t"
1145                 "psrlq  $8, %%mm6\n\t"
1146                 "psrlq  $8, %%mm7\n\t"
1147                 "pand   %2, %%mm0\n\t"
1148                 "pand   %2, %%mm1\n\t"
1149                 "pand   %2, %%mm4\n\t"
1150                 "pand   %2, %%mm5\n\t"
1151                 "pand   %3, %%mm2\n\t"
1152                 "pand   %3, %%mm3\n\t"
1153                 "pand   %3, %%mm6\n\t"
1154                 "pand   %3, %%mm7\n\t"
1155                 "por    %%mm2, %%mm0\n\t"
1156                 "por    %%mm3, %%mm1\n\t"
1157                 "por    %%mm6, %%mm4\n\t"
1158                 "por    %%mm7, %%mm5\n\t"
1159
1160                 "movq   %%mm1, %%mm2\n\t"
1161                 "movq   %%mm4, %%mm3\n\t"
1162                 "psllq  $48, %%mm2\n\t"
1163                 "psllq  $32, %%mm3\n\t"
1164                 "pand   %4, %%mm2\n\t"
1165                 "pand   %5, %%mm3\n\t"
1166                 "por    %%mm2, %%mm0\n\t"
1167                 "psrlq  $16, %%mm1\n\t"
1168                 "psrlq  $32, %%mm4\n\t"
1169                 "psllq  $16, %%mm5\n\t"
1170                 "por    %%mm3, %%mm1\n\t"
1171                 "pand   %6, %%mm5\n\t"
1172                 "por    %%mm5, %%mm4\n\t"
1173
1174                 MOVNTQ" %%mm0, %0\n\t"
1175                 MOVNTQ" %%mm1, 8%0\n\t"
1176                 MOVNTQ" %%mm4, 16%0"
1177
1178                 :"=m"(*d)
1179                 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1180                 :"memory");
1181                 d += 24;
1182                 s += 8;
1183         }
1184         __asm __volatile(SFENCE:::"memory");
1185         __asm __volatile(EMMS:::"memory");
1186 #endif
1187         while(s < end)
1188         {
1189                 register uint16_t bgr;
1190                 bgr = *s++;
1191                 *d++ = (bgr&0x1F)<<3;
1192                 *d++ = (bgr&0x7E0)>>3;
1193                 *d++ = (bgr&0xF800)>>8;
1194         }
1195 }
1196
1197 static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_size)
1198 {
1199         const uint16_t *end;
1200 #ifdef HAVE_MMX
1201         const uint16_t *mm_end;
1202 #endif
1203         uint8_t *d = (uint8_t *)dst;
1204         const uint16_t *s = (const uint16_t *)src;
1205         end = s + src_size/2;
1206 #ifdef HAVE_MMX
1207         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1208         __asm __volatile("pxor  %%mm7,%%mm7\n\t":::"memory");
1209         mm_end = end - 3;
1210         while(s < mm_end)
1211         {
1212             __asm __volatile(
1213                 PREFETCH" 32%1\n\t"
1214                 "movq   %1, %%mm0\n\t"
1215                 "movq   %1, %%mm1\n\t"
1216                 "movq   %1, %%mm2\n\t"
1217                 "pand   %2, %%mm0\n\t"
1218                 "pand   %3, %%mm1\n\t"
1219                 "pand   %4, %%mm2\n\t"
1220                 "psllq  $3, %%mm0\n\t"
1221                 "psrlq  $2, %%mm1\n\t"
1222                 "psrlq  $7, %%mm2\n\t"
1223                 "movq   %%mm0, %%mm3\n\t"
1224                 "movq   %%mm1, %%mm4\n\t"
1225                 "movq   %%mm2, %%mm5\n\t"
1226                 "punpcklwd %%mm7, %%mm0\n\t"
1227                 "punpcklwd %%mm7, %%mm1\n\t"
1228                 "punpcklwd %%mm7, %%mm2\n\t"
1229                 "punpckhwd %%mm7, %%mm3\n\t"
1230                 "punpckhwd %%mm7, %%mm4\n\t"
1231                 "punpckhwd %%mm7, %%mm5\n\t"
1232                 "psllq  $8, %%mm1\n\t"
1233                 "psllq  $16, %%mm2\n\t"
1234                 "por    %%mm1, %%mm0\n\t"
1235                 "por    %%mm2, %%mm0\n\t"
1236                 "psllq  $8, %%mm4\n\t"
1237                 "psllq  $16, %%mm5\n\t"
1238                 "por    %%mm4, %%mm3\n\t"
1239                 "por    %%mm5, %%mm3\n\t"
1240                 MOVNTQ" %%mm0, %0\n\t"
1241                 MOVNTQ" %%mm3, 8%0\n\t"
1242                 :"=m"(*d)
1243                 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r)
1244                 :"memory");
1245                 d += 16;
1246                 s += 4;
1247         }
1248         __asm __volatile(SFENCE:::"memory");
1249         __asm __volatile(EMMS:::"memory");
1250 #endif
1251         while(s < end)
1252         {
1253 #if 0 //slightly slower on athlon
1254                 int bgr= *s++;
1255                 *((uint32_t*)d)++ = ((bgr&0x1F)<<3) + ((bgr&0x3E0)<<6) + ((bgr&0x7C00)<<9);
1256 #else
1257                 register uint16_t bgr;
1258                 bgr = *s++;
1259 #ifdef WORDS_BIGENDIAN
1260                 *d++ = 0;
1261                 *d++ = (bgr&0x7C00)>>7;
1262                 *d++ = (bgr&0x3E0)>>2;
1263                 *d++ = (bgr&0x1F)<<3;
1264 #else
1265                 *d++ = (bgr&0x1F)<<3;
1266                 *d++ = (bgr&0x3E0)>>2;
1267                 *d++ = (bgr&0x7C00)>>7;
1268                 *d++ = 0;
1269 #endif
1270
1271 #endif
1272         }
1273 }
1274
1275 static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_size)
1276 {
1277         const uint16_t *end;
1278 #ifdef HAVE_MMX
1279         const uint16_t *mm_end;
1280 #endif
1281         uint8_t *d = (uint8_t *)dst;
1282         const uint16_t *s = (uint16_t *)src;
1283         end = s + src_size/2;
1284 #ifdef HAVE_MMX
1285         __asm __volatile(PREFETCH"      %0"::"m"(*s):"memory");
1286         __asm __volatile("pxor  %%mm7,%%mm7\n\t":::"memory");
1287         mm_end = end - 3;
1288         while(s < mm_end)
1289         {
1290             __asm __volatile(
1291                 PREFETCH" 32%1\n\t"
1292                 "movq   %1, %%mm0\n\t"
1293                 "movq   %1, %%mm1\n\t"
1294                 "movq   %1, %%mm2\n\t"
1295                 "pand   %2, %%mm0\n\t"
1296                 "pand   %3, %%mm1\n\t"
1297                 "pand   %4, %%mm2\n\t"
1298                 "psllq  $3, %%mm0\n\t"
1299                 "psrlq  $3, %%mm1\n\t"
1300                 "psrlq  $8, %%mm2\n\t"
1301                 "movq   %%mm0, %%mm3\n\t"
1302                 "movq   %%mm1, %%mm4\n\t"
1303                 "movq   %%mm2, %%mm5\n\t"
1304                 "punpcklwd %%mm7, %%mm0\n\t"
1305                 "punpcklwd %%mm7, %%mm1\n\t"
1306                 "punpcklwd %%mm7, %%mm2\n\t"
1307                 "punpckhwd %%mm7, %%mm3\n\t"
1308                 "punpckhwd %%mm7, %%mm4\n\t"
1309                 "punpckhwd %%mm7, %%mm5\n\t"
1310                 "psllq  $8, %%mm1\n\t"
1311                 "psllq  $16, %%mm2\n\t"
1312                 "por    %%mm1, %%mm0\n\t"
1313                 "por    %%mm2, %%mm0\n\t"
1314                 "psllq  $8, %%mm4\n\t"
1315                 "psllq  $16, %%mm5\n\t"
1316                 "por    %%mm4, %%mm3\n\t"
1317                 "por    %%mm5, %%mm3\n\t"
1318                 MOVNTQ" %%mm0, %0\n\t"
1319                 MOVNTQ" %%mm3, 8%0\n\t"
1320                 :"=m"(*d)
1321                 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r)
1322                 :"memory");
1323                 d += 16;
1324                 s += 4;
1325         }
1326         __asm __volatile(SFENCE:::"memory");
1327         __asm __volatile(EMMS:::"memory");
1328 #endif
1329         while(s < end)
1330         {
1331                 register uint16_t bgr;
1332                 bgr = *s++;
1333 #ifdef WORDS_BIGENDIAN
1334                 *d++ = 0;
1335                 *d++ = (bgr&0xF800)>>8;
1336                 *d++ = (bgr&0x7E0)>>3;
1337                 *d++ = (bgr&0x1F)<<3;
1338 #else
1339                 *d++ = (bgr&0x1F)<<3;
1340                 *d++ = (bgr&0x7E0)>>3;
1341                 *d++ = (bgr&0xF800)>>8;
1342                 *d++ = 0;
1343 #endif
1344         }
1345 }
1346
1347 static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, long src_size)
1348 {
1349 #ifdef HAVE_MMX
1350 /* TODO: unroll this loop */
1351         asm volatile (
1352                 "xor %%"REG_a", %%"REG_a"       \n\t"
1353                 ".balign 16                     \n\t"
1354                 "1:                             \n\t"
1355                 PREFETCH" 32(%0, %%"REG_a")     \n\t"
1356                 "movq (%0, %%"REG_a"), %%mm0    \n\t"
1357                 "movq %%mm0, %%mm1              \n\t"
1358                 "movq %%mm0, %%mm2              \n\t"
1359                 "pslld $16, %%mm0               \n\t"
1360                 "psrld $16, %%mm1               \n\t"
1361                 "pand "MANGLE(mask32r)", %%mm0  \n\t"
1362                 "pand "MANGLE(mask32g)", %%mm2  \n\t"
1363                 "pand "MANGLE(mask32b)", %%mm1  \n\t"
1364                 "por %%mm0, %%mm2               \n\t"
1365                 "por %%mm1, %%mm2               \n\t"
1366                 MOVNTQ" %%mm2, (%1, %%"REG_a")  \n\t"
1367                 "add $8, %%"REG_a"              \n\t"
1368                 "cmp %2, %%"REG_a"              \n\t"
1369                 " jb 1b                         \n\t"
1370                 :: "r" (src), "r"(dst), "r" (src_size-7)
1371                 : "%"REG_a
1372         );
1373
1374         __asm __volatile(SFENCE:::"memory");
1375         __asm __volatile(EMMS:::"memory");
1376 #else
1377         unsigned i;
1378         unsigned num_pixels = src_size >> 2;
1379         for(i=0; i<num_pixels; i++)
1380         {
1381 #ifdef WORDS_BIGENDIAN  
1382           dst[4*i + 1] = src[4*i + 3];
1383           dst[4*i + 2] = src[4*i + 2];
1384           dst[4*i + 3] = src[4*i + 1];
1385 #else
1386           dst[4*i + 0] = src[4*i + 2];
1387           dst[4*i + 1] = src[4*i + 1];
1388           dst[4*i + 2] = src[4*i + 0];
1389 #endif
1390         }
1391 #endif
1392 }
1393
1394 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, long src_size)
1395 {
1396         unsigned i;
1397 #ifdef HAVE_MMX
1398         long mmx_size= 23 - src_size;
1399         asm volatile (
1400                 "movq "MANGLE(mask24r)", %%mm5  \n\t"
1401                 "movq "MANGLE(mask24g)", %%mm6  \n\t"
1402                 "movq "MANGLE(mask24b)", %%mm7  \n\t"
1403                 ".balign 16                     \n\t"
1404                 "1:                             \n\t"
1405                 PREFETCH" 32(%1, %%"REG_a")     \n\t"
1406                 "movq   (%1, %%"REG_a"), %%mm0  \n\t" // BGR BGR BG
1407                 "movq   (%1, %%"REG_a"), %%mm1  \n\t" // BGR BGR BG
1408                 "movq  2(%1, %%"REG_a"), %%mm2  \n\t" // R BGR BGR B
1409                 "psllq $16, %%mm0               \n\t" // 00 BGR BGR
1410                 "pand %%mm5, %%mm0              \n\t"
1411                 "pand %%mm6, %%mm1              \n\t"
1412                 "pand %%mm7, %%mm2              \n\t"
1413                 "por %%mm0, %%mm1               \n\t"
1414                 "por %%mm2, %%mm1               \n\t"                
1415                 "movq  6(%1, %%"REG_a"), %%mm0  \n\t" // BGR BGR BG
1416                 MOVNTQ" %%mm1,   (%2, %%"REG_a")\n\t" // RGB RGB RG
1417                 "movq  8(%1, %%"REG_a"), %%mm1  \n\t" // R BGR BGR B
1418                 "movq 10(%1, %%"REG_a"), %%mm2  \n\t" // GR BGR BGR
1419                 "pand %%mm7, %%mm0              \n\t"
1420                 "pand %%mm5, %%mm1              \n\t"
1421                 "pand %%mm6, %%mm2              \n\t"
1422                 "por %%mm0, %%mm1               \n\t"
1423                 "por %%mm2, %%mm1               \n\t"                
1424                 "movq 14(%1, %%"REG_a"), %%mm0  \n\t" // R BGR BGR B
1425                 MOVNTQ" %%mm1,  8(%2, %%"REG_a")\n\t" // B RGB RGB R
1426                 "movq 16(%1, %%"REG_a"), %%mm1  \n\t" // GR BGR BGR
1427                 "movq 18(%1, %%"REG_a"), %%mm2  \n\t" // BGR BGR BG
1428                 "pand %%mm6, %%mm0              \n\t"
1429                 "pand %%mm7, %%mm1              \n\t"
1430                 "pand %%mm5, %%mm2              \n\t"
1431                 "por %%mm0, %%mm1               \n\t"
1432                 "por %%mm2, %%mm1               \n\t"                
1433                 MOVNTQ" %%mm1, 16(%2, %%"REG_a")\n\t"
1434                 "add $24, %%"REG_a"             \n\t"
1435                 " js 1b                         \n\t"
1436                 : "+a" (mmx_size)
1437                 : "r" (src-mmx_size), "r"(dst-mmx_size)
1438         );
1439
1440         __asm __volatile(SFENCE:::"memory");
1441         __asm __volatile(EMMS:::"memory");
1442
1443         if(mmx_size==23) return; //finihsed, was multiple of 8
1444
1445         src+= src_size;
1446         dst+= src_size;
1447         src_size= 23-mmx_size;
1448         src-= src_size;
1449         dst-= src_size;
1450 #endif
1451         for(i=0; i<src_size; i+=3)
1452         {
1453                 register uint8_t x;
1454                 x          = src[i + 2];
1455                 dst[i + 1] = src[i + 1];
1456                 dst[i + 2] = src[i + 0];
1457                 dst[i + 0] = x;
1458         }
1459 }
1460
1461 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1462         long width, long height,
1463         long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1464 {
1465         long y;
1466         const long chromWidth= width>>1;
1467         for(y=0; y<height; y++)
1468         {
1469 #ifdef HAVE_MMX
1470 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1471                 asm volatile(
1472                         "xor %%"REG_a", %%"REG_a"       \n\t"
1473                         ".balign 16                     \n\t"
1474                         "1:                             \n\t"
1475                         PREFETCH" 32(%1, %%"REG_a", 2)  \n\t"
1476                         PREFETCH" 32(%2, %%"REG_a")     \n\t"
1477                         PREFETCH" 32(%3, %%"REG_a")     \n\t"
1478                         "movq (%2, %%"REG_a"), %%mm0    \n\t" // U(0)
1479                         "movq %%mm0, %%mm2              \n\t" // U(0)
1480                         "movq (%3, %%"REG_a"), %%mm1    \n\t" // V(0)
1481                         "punpcklbw %%mm1, %%mm0         \n\t" // UVUV UVUV(0)
1482                         "punpckhbw %%mm1, %%mm2         \n\t" // UVUV UVUV(8)
1483
1484                         "movq (%1, %%"REG_a",2), %%mm3  \n\t" // Y(0)
1485                         "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1486                         "movq %%mm3, %%mm4              \n\t" // Y(0)
1487                         "movq %%mm5, %%mm6              \n\t" // Y(8)
1488                         "punpcklbw %%mm0, %%mm3         \n\t" // YUYV YUYV(0)
1489                         "punpckhbw %%mm0, %%mm4         \n\t" // YUYV YUYV(4)
1490                         "punpcklbw %%mm2, %%mm5         \n\t" // YUYV YUYV(8)
1491                         "punpckhbw %%mm2, %%mm6         \n\t" // YUYV YUYV(12)
1492
1493                         MOVNTQ" %%mm3, (%0, %%"REG_a", 4)\n\t"
1494                         MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1495                         MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4)\n\t"
1496                         MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1497
1498                         "add $8, %%"REG_a"              \n\t"
1499                         "cmp %4, %%"REG_a"              \n\t"
1500                         " jb 1b                         \n\t"
1501                         ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1502                         : "%"REG_a
1503                 );
1504 #else
1505
1506 #if defined ARCH_ALPHA && defined HAVE_MVI
1507 #define pl2yuy2(n)                                      \
1508         y1 = yc[n];                                     \
1509         y2 = yc2[n];                                    \
1510         u = uc[n];                                      \
1511         v = vc[n];                                      \
1512         asm("unpkbw %1, %0" : "=r"(y1) : "r"(y1));      \
1513         asm("unpkbw %1, %0" : "=r"(y2) : "r"(y2));      \
1514         asm("unpkbl %1, %0" : "=r"(u) : "r"(u));        \
1515         asm("unpkbl %1, %0" : "=r"(v) : "r"(v));        \
1516         yuv1 = (u << 8) + (v << 24);                    \
1517         yuv2 = yuv1 + y2;                               \
1518         yuv1 += y1;                                     \
1519         qdst[n] = yuv1;                                 \
1520         qdst2[n] = yuv2;
1521
1522                 int i;
1523                 uint64_t *qdst = (uint64_t *) dst;
1524                 uint64_t *qdst2 = (uint64_t *) (dst + dstStride);
1525                 const uint32_t *yc = (uint32_t *) ysrc;
1526                 const uint32_t *yc2 = (uint32_t *) (ysrc + lumStride);
1527                 const uint16_t *uc = (uint16_t*) usrc, *vc = (uint16_t*) vsrc;
1528                 for(i = 0; i < chromWidth; i += 8){
1529                         uint64_t y1, y2, yuv1, yuv2;
1530                         uint64_t u, v;
1531                         /* Prefetch */
1532                         asm("ldq $31,64(%0)" :: "r"(yc));
1533                         asm("ldq $31,64(%0)" :: "r"(yc2));
1534                         asm("ldq $31,64(%0)" :: "r"(uc));
1535                         asm("ldq $31,64(%0)" :: "r"(vc));
1536
1537                         pl2yuy2(0);
1538                         pl2yuy2(1);
1539                         pl2yuy2(2);
1540                         pl2yuy2(3);
1541
1542                         yc += 4;
1543                         yc2 += 4;
1544                         uc += 4;
1545                         vc += 4;
1546                         qdst += 4;
1547                         qdst2 += 4;
1548                 }
1549                 y++;
1550                 ysrc += lumStride;
1551                 dst += dstStride;
1552
1553 #elif __WORDSIZE >= 64
1554                 int i;
1555                 uint64_t *ldst = (uint64_t *) dst;
1556                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1557                 for(i = 0; i < chromWidth; i += 2){
1558                         uint64_t k, l;
1559                         k = yc[0] + (uc[0] << 8) +
1560                             (yc[1] << 16) + (vc[0] << 24);
1561                         l = yc[2] + (uc[1] << 8) +
1562                             (yc[3] << 16) + (vc[1] << 24);
1563                         *ldst++ = k + (l << 32);
1564                         yc += 4;
1565                         uc += 2;
1566                         vc += 2;
1567                 }
1568
1569 #else
1570                 int i, *idst = (int32_t *) dst;
1571                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1572                 for(i = 0; i < chromWidth; i++){
1573 #ifdef WORDS_BIGENDIAN
1574                         *idst++ = (yc[0] << 24)+ (uc[0] << 16) +
1575                             (yc[1] << 8) + (vc[0] << 0);
1576 #else
1577                         *idst++ = yc[0] + (uc[0] << 8) +
1578                             (yc[1] << 16) + (vc[0] << 24);
1579 #endif
1580                         yc += 2;
1581                         uc++;
1582                         vc++;
1583                 }
1584 #endif
1585 #endif
1586                 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1587                 {
1588                         usrc += chromStride;
1589                         vsrc += chromStride;
1590                 }
1591                 ysrc += lumStride;
1592                 dst += dstStride;
1593         }
1594 #ifdef HAVE_MMX
1595 asm(    EMMS" \n\t"
1596         SFENCE" \n\t"
1597         :::"memory");
1598 #endif
1599 }
1600
1601 /**
1602  *
1603  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1604  * problem for anyone then tell me, and ill fix it)
1605  */
1606 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1607         long width, long height,
1608         long lumStride, long chromStride, long dstStride)
1609 {
1610         //FIXME interpolate chroma
1611         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1612 }
1613
1614 static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1615         long width, long height,
1616         long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1617 {
1618         long y;
1619         const long chromWidth= width>>1;
1620         for(y=0; y<height; y++)
1621         {
1622 #ifdef HAVE_MMX
1623 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1624                 asm volatile(
1625                         "xor %%"REG_a", %%"REG_a"       \n\t"
1626                         ".balign 16                     \n\t"
1627                         "1:                             \n\t"
1628                         PREFETCH" 32(%1, %%"REG_a", 2)  \n\t"
1629                         PREFETCH" 32(%2, %%"REG_a")     \n\t"
1630                         PREFETCH" 32(%3, %%"REG_a")     \n\t"
1631                         "movq (%2, %%"REG_a"), %%mm0    \n\t" // U(0)
1632                         "movq %%mm0, %%mm2              \n\t" // U(0)
1633                         "movq (%3, %%"REG_a"), %%mm1    \n\t" // V(0)
1634                         "punpcklbw %%mm1, %%mm0         \n\t" // UVUV UVUV(0)
1635                         "punpckhbw %%mm1, %%mm2         \n\t" // UVUV UVUV(8)
1636
1637                         "movq (%1, %%"REG_a",2), %%mm3  \n\t" // Y(0)
1638                         "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1639                         "movq %%mm0, %%mm4              \n\t" // Y(0)
1640                         "movq %%mm2, %%mm6              \n\t" // Y(8)
1641                         "punpcklbw %%mm3, %%mm0         \n\t" // YUYV YUYV(0)
1642                         "punpckhbw %%mm3, %%mm4         \n\t" // YUYV YUYV(4)
1643                         "punpcklbw %%mm5, %%mm2         \n\t" // YUYV YUYV(8)
1644                         "punpckhbw %%mm5, %%mm6         \n\t" // YUYV YUYV(12)
1645
1646                         MOVNTQ" %%mm0, (%0, %%"REG_a", 4)\n\t"
1647                         MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1648                         MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4)\n\t"
1649                         MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1650
1651                         "add $8, %%"REG_a"              \n\t"
1652                         "cmp %4, %%"REG_a"              \n\t"
1653                         " jb 1b                         \n\t"
1654                         ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1655                         : "%"REG_a
1656                 );
1657 #else
1658 //FIXME adapt the alpha asm code from yv12->yuy2
1659
1660 #if __WORDSIZE >= 64
1661                 int i;
1662                 uint64_t *ldst = (uint64_t *) dst;
1663                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1664                 for(i = 0; i < chromWidth; i += 2){
1665                         uint64_t k, l;
1666                         k = uc[0] + (yc[0] << 8) +
1667                             (vc[0] << 16) + (yc[1] << 24);
1668                         l = uc[1] + (yc[2] << 8) +
1669                             (vc[1] << 16) + (yc[3] << 24);
1670                         *ldst++ = k + (l << 32);
1671                         yc += 4;
1672                         uc += 2;
1673                         vc += 2;
1674                 }
1675
1676 #else
1677                 int i, *idst = (int32_t *) dst;
1678                 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1679                 for(i = 0; i < chromWidth; i++){
1680 #ifdef WORDS_BIGENDIAN
1681                         *idst++ = (uc[0] << 24)+ (yc[0] << 16) +
1682                             (vc[0] << 8) + (yc[1] << 0);
1683 #else
1684                         *idst++ = uc[0] + (yc[0] << 8) +
1685                             (vc[0] << 16) + (yc[1] << 24);
1686 #endif
1687                         yc += 2;
1688                         uc++;
1689                         vc++;
1690                 }
1691 #endif
1692 #endif
1693                 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1694                 {
1695                         usrc += chromStride;
1696                         vsrc += chromStride;
1697                 }
1698                 ysrc += lumStride;
1699                 dst += dstStride;
1700         }
1701 #ifdef HAVE_MMX
1702 asm(    EMMS" \n\t"
1703         SFENCE" \n\t"
1704         :::"memory");
1705 #endif
1706 }
1707
1708 /**
1709  *
1710  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1711  * problem for anyone then tell me, and ill fix it)
1712  */
1713 static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1714         long width, long height,
1715         long lumStride, long chromStride, long dstStride)
1716 {
1717         //FIXME interpolate chroma
1718         RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1719 }
1720
1721 /**
1722  *
1723  * width should be a multiple of 16
1724  */
1725 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1726         long width, long height,
1727         long lumStride, long chromStride, long dstStride)
1728 {
1729         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1730 }
1731
1732 /**
1733  *
1734  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1735  * problem for anyone then tell me, and ill fix it)
1736  */
1737 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1738         long width, long height,
1739         long lumStride, long chromStride, long srcStride)
1740 {
1741         long y;
1742         const long chromWidth= width>>1;
1743         for(y=0; y<height; y+=2)
1744         {
1745 #ifdef HAVE_MMX
1746                 asm volatile(
1747                         "xor %%"REG_a", %%"REG_a"       \n\t"
1748                         "pcmpeqw %%mm7, %%mm7           \n\t"
1749                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
1750                         ".balign 16                     \n\t"
1751                         "1:                             \n\t"
1752                         PREFETCH" 64(%0, %%"REG_a", 4)  \n\t"
1753                         "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1754                         "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1755                         "movq %%mm0, %%mm2              \n\t" // YUYV YUYV(0)
1756                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(4)
1757                         "psrlw $8, %%mm0                \n\t" // U0V0 U0V0(0)
1758                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(4)
1759                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(0)
1760                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(4)
1761                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
1762                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
1763
1764                         MOVNTQ" %%mm2, (%1, %%"REG_a", 2)\n\t"
1765
1766                         "movq 16(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(8)
1767                         "movq 24(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(12)
1768                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(8)
1769                         "movq %%mm2, %%mm4              \n\t" // YUYV YUYV(12)
1770                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(8)
1771                         "psrlw $8, %%mm2                \n\t" // U0V0 U0V0(12)
1772                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(8)
1773                         "pand %%mm7, %%mm4              \n\t" // Y0Y0 Y0Y0(12)
1774                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
1775                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
1776
1777                         MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2)\n\t"
1778
1779                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
1780                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
1781                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
1782                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
1783                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
1784                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
1785                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
1786                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
1787
1788                         MOVNTQ" %%mm0, (%3, %%"REG_a")  \n\t"
1789                         MOVNTQ" %%mm2, (%2, %%"REG_a")  \n\t"
1790
1791                         "add $8, %%"REG_a"              \n\t"
1792                         "cmp %4, %%"REG_a"              \n\t"
1793                         " jb 1b                         \n\t"
1794                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1795                         : "memory", "%"REG_a
1796                 );
1797
1798                 ydst += lumStride;
1799                 src  += srcStride;
1800
1801                 asm volatile(
1802                         "xor %%"REG_a", %%"REG_a"       \n\t"
1803                         ".balign 16                     \n\t"
1804                         "1:                             \n\t"
1805                         PREFETCH" 64(%0, %%"REG_a", 4)  \n\t"
1806                         "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1807                         "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1808                         "movq 16(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(8)
1809                         "movq 24(%0, %%"REG_a", 4), %%mm3\n\t" // YUYV YUYV(12)
1810                         "pand %%mm7, %%mm0              \n\t" // Y0Y0 Y0Y0(0)
1811                         "pand %%mm7, %%mm1              \n\t" // Y0Y0 Y0Y0(4)
1812                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(8)
1813                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(12)
1814                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
1815                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
1816
1817                         MOVNTQ" %%mm0, (%1, %%"REG_a", 2)\n\t"
1818                         MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2)\n\t"
1819
1820                         "add $8, %%"REG_a"              \n\t"
1821                         "cmp %4, %%"REG_a"              \n\t"
1822                         " jb 1b                         \n\t"
1823
1824                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1825                         : "memory", "%"REG_a
1826                 );
1827 #else
1828                 long i;
1829                 for(i=0; i<chromWidth; i++)
1830                 {
1831                         ydst[2*i+0]     = src[4*i+0];
1832                         udst[i]         = src[4*i+1];
1833                         ydst[2*i+1]     = src[4*i+2];
1834                         vdst[i]         = src[4*i+3];
1835                 }
1836                 ydst += lumStride;
1837                 src  += srcStride;
1838
1839                 for(i=0; i<chromWidth; i++)
1840                 {
1841                         ydst[2*i+0]     = src[4*i+0];
1842                         ydst[2*i+1]     = src[4*i+2];
1843                 }
1844 #endif
1845                 udst += chromStride;
1846                 vdst += chromStride;
1847                 ydst += lumStride;
1848                 src  += srcStride;
1849         }
1850 #ifdef HAVE_MMX
1851 asm volatile(   EMMS" \n\t"
1852                 SFENCE" \n\t"
1853                 :::"memory");
1854 #endif
1855 }
1856
1857 static inline void RENAME(yvu9toyv12)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,
1858         uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1859         long width, long height, long lumStride, long chromStride)
1860 {
1861         /* Y Plane */
1862         memcpy(ydst, ysrc, width*height);
1863
1864         /* XXX: implement upscaling for U,V */
1865 }
1866
1867 static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, long srcWidth, long srcHeight, long srcStride, long dstStride)
1868 {
1869         long x,y;
1870         
1871         dst[0]= src[0];
1872         
1873         // first line
1874         for(x=0; x<srcWidth-1; x++){
1875                 dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
1876                 dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
1877         }
1878         dst[2*srcWidth-1]= src[srcWidth-1];
1879         
1880         dst+= dstStride;
1881
1882         for(y=1; y<srcHeight; y++){
1883 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
1884                 const long mmxSize= srcWidth&~15;
1885                 asm volatile(
1886                         "mov %4, %%"REG_a"              \n\t"
1887                         "1:                             \n\t"
1888                         "movq (%0, %%"REG_a"), %%mm0    \n\t"
1889                         "movq (%1, %%"REG_a"), %%mm1    \n\t"
1890                         "movq 1(%0, %%"REG_a"), %%mm2   \n\t"
1891                         "movq 1(%1, %%"REG_a"), %%mm3   \n\t"
1892                         "movq -1(%0, %%"REG_a"), %%mm4  \n\t"
1893                         "movq -1(%1, %%"REG_a"), %%mm5  \n\t"
1894                         PAVGB" %%mm0, %%mm5             \n\t"
1895                         PAVGB" %%mm0, %%mm3             \n\t"
1896                         PAVGB" %%mm0, %%mm5             \n\t"
1897                         PAVGB" %%mm0, %%mm3             \n\t"
1898                         PAVGB" %%mm1, %%mm4             \n\t"
1899                         PAVGB" %%mm1, %%mm2             \n\t"
1900                         PAVGB" %%mm1, %%mm4             \n\t"
1901                         PAVGB" %%mm1, %%mm2             \n\t"
1902                         "movq %%mm5, %%mm7              \n\t"
1903                         "movq %%mm4, %%mm6              \n\t"
1904                         "punpcklbw %%mm3, %%mm5         \n\t"
1905                         "punpckhbw %%mm3, %%mm7         \n\t"
1906                         "punpcklbw %%mm2, %%mm4         \n\t"
1907                         "punpckhbw %%mm2, %%mm6         \n\t"
1908 #if 1
1909                         MOVNTQ" %%mm5, (%2, %%"REG_a", 2)\n\t"
1910                         MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1911                         MOVNTQ" %%mm4, (%3, %%"REG_a", 2)\n\t"
1912                         MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1913 #else
1914                         "movq %%mm5, (%2, %%"REG_a", 2) \n\t"
1915                         "movq %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1916                         "movq %%mm4, (%3, %%"REG_a", 2) \n\t"
1917                         "movq %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1918 #endif
1919                         "add $8, %%"REG_a"              \n\t"
1920                         " js 1b                         \n\t"
1921                         :: "r" (src + mmxSize  ), "r" (src + srcStride + mmxSize  ),
1922                            "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
1923                            "g" (-mmxSize)
1924                         : "%"REG_a
1925
1926                 );
1927 #else
1928                 const long mmxSize=1;
1929 #endif
1930                 dst[0        ]= (3*src[0] +   src[srcStride])>>2;
1931                 dst[dstStride]= (  src[0] + 3*src[srcStride])>>2;
1932
1933                 for(x=mmxSize-1; x<srcWidth-1; x++){
1934                         dst[2*x          +1]= (3*src[x+0] +   src[x+srcStride+1])>>2;
1935                         dst[2*x+dstStride+2]= (  src[x+0] + 3*src[x+srcStride+1])>>2;
1936                         dst[2*x+dstStride+1]= (  src[x+1] + 3*src[x+srcStride  ])>>2;
1937                         dst[2*x          +2]= (3*src[x+1] +   src[x+srcStride  ])>>2;
1938                 }
1939                 dst[srcWidth*2 -1            ]= (3*src[srcWidth-1] +   src[srcWidth-1 + srcStride])>>2;
1940                 dst[srcWidth*2 -1 + dstStride]= (  src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
1941
1942                 dst+=dstStride*2;
1943                 src+=srcStride;
1944         }
1945         
1946         // last line
1947 #if 1
1948         dst[0]= src[0];
1949         
1950         for(x=0; x<srcWidth-1; x++){
1951                 dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
1952                 dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
1953         }
1954         dst[2*srcWidth-1]= src[srcWidth-1];
1955 #else
1956         for(x=0; x<srcWidth; x++){
1957                 dst[2*x+0]=
1958                 dst[2*x+1]= src[x];
1959         }
1960 #endif
1961
1962 #ifdef HAVE_MMX
1963 asm volatile(   EMMS" \n\t"
1964                 SFENCE" \n\t"
1965                 :::"memory");
1966 #endif
1967 }
1968
1969 /**
1970  *
1971  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1972  * problem for anyone then tell me, and ill fix it)
1973  * chrominance data is only taken from every secound line others are ignored FIXME write HQ version
1974  */
1975 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1976         long width, long height,
1977         long lumStride, long chromStride, long srcStride)
1978 {
1979         long y;
1980         const long chromWidth= width>>1;
1981         for(y=0; y<height; y+=2)
1982         {
1983 #ifdef HAVE_MMX
1984                 asm volatile(
1985                         "xorl %%eax, %%eax              \n\t"
1986                         "pcmpeqw %%mm7, %%mm7           \n\t"
1987                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
1988                         ".balign 16                     \n\t"
1989                         "1:                             \n\t"
1990                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
1991                         "movq (%0, %%eax, 4), %%mm0     \n\t" // UYVY UYVY(0)
1992                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // UYVY UYVY(4)
1993                         "movq %%mm0, %%mm2              \n\t" // UYVY UYVY(0)
1994                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(4)
1995                         "pand %%mm7, %%mm0              \n\t" // U0V0 U0V0(0)
1996                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(4)
1997                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(0)
1998                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(4)
1999                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
2000                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
2001
2002                         MOVNTQ" %%mm2, (%1, %%eax, 2)   \n\t"
2003
2004                         "movq 16(%0, %%eax, 4), %%mm1   \n\t" // UYVY UYVY(8)
2005                         "movq 24(%0, %%eax, 4), %%mm2   \n\t" // UYVY UYVY(12)
2006                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(8)
2007                         "movq %%mm2, %%mm4              \n\t" // UYVY UYVY(12)
2008                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(8)
2009                         "pand %%mm7, %%mm2              \n\t" // U0V0 U0V0(12)
2010                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(8)
2011                         "psrlw $8, %%mm4                \n\t" // Y0Y0 Y0Y0(12)
2012                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
2013                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
2014
2015                         MOVNTQ" %%mm3, 8(%1, %%eax, 2)  \n\t"
2016
2017                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
2018                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
2019                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
2020                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
2021                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
2022                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
2023                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
2024                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
2025
2026                         MOVNTQ" %%mm0, (%3, %%eax)      \n\t"
2027                         MOVNTQ" %%mm2, (%2, %%eax)      \n\t"
2028
2029                         "addl $8, %%eax                 \n\t"
2030                         "cmpl %4, %%eax                 \n\t"
2031                         " jb 1b                         \n\t"
2032                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2033                         : "memory", "%eax"
2034                 );
2035
2036                 ydst += lumStride;
2037                 src  += srcStride;
2038
2039                 asm volatile(
2040                         "xorl %%eax, %%eax              \n\t"
2041                         ".balign 16                     \n\t"
2042                         "1:                             \n\t"
2043                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
2044                         "movq (%0, %%eax, 4), %%mm0     \n\t" // YUYV YUYV(0)
2045                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // YUYV YUYV(4)
2046                         "movq 16(%0, %%eax, 4), %%mm2   \n\t" // YUYV YUYV(8)
2047                         "movq 24(%0, %%eax, 4), %%mm3   \n\t" // YUYV YUYV(12)
2048                         "psrlw $8, %%mm0                \n\t" // Y0Y0 Y0Y0(0)
2049                         "psrlw $8, %%mm1                \n\t" // Y0Y0 Y0Y0(4)
2050                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(8)
2051                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(12)
2052                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
2053                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
2054
2055                         MOVNTQ" %%mm0, (%1, %%eax, 2)   \n\t"
2056                         MOVNTQ" %%mm2, 8(%1, %%eax, 2)  \n\t"
2057
2058                         "addl $8, %%eax                 \n\t"
2059                         "cmpl %4, %%eax                 \n\t"
2060                         " jb 1b                         \n\t"
2061
2062                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2063                         : "memory", "%eax"
2064                 );
2065 #else
2066                 long i;
2067                 for(i=0; i<chromWidth; i++)
2068                 {
2069                         udst[i]         = src[4*i+0];
2070                         ydst[2*i+0]     = src[4*i+1];
2071                         vdst[i]         = src[4*i+2];
2072                         ydst[2*i+1]     = src[4*i+3];
2073                 }
2074                 ydst += lumStride;
2075                 src  += srcStride;
2076
2077                 for(i=0; i<chromWidth; i++)
2078                 {
2079                         ydst[2*i+0]     = src[4*i+1];
2080                         ydst[2*i+1]     = src[4*i+3];
2081                 }
2082 #endif
2083                 udst += chromStride;
2084                 vdst += chromStride;
2085                 ydst += lumStride;
2086                 src  += srcStride;
2087         }
2088 #ifdef HAVE_MMX
2089 asm volatile(   EMMS" \n\t"
2090                 SFENCE" \n\t"
2091                 :::"memory");
2092 #endif
2093 }
2094
2095 /**
2096  *
2097  * height should be a multiple of 2 and width should be a multiple of 2 (if this is a
2098  * problem for anyone then tell me, and ill fix it)
2099  * chrominance data is only taken from every secound line others are ignored in the C version FIXME write HQ version
2100  */
2101 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2102         long width, long height,
2103         long lumStride, long chromStride, long srcStride)
2104 {
2105         long y;
2106         const long chromWidth= width>>1;
2107 #ifdef HAVE_MMX
2108         for(y=0; y<height-2; y+=2)
2109         {
2110                 long i;
2111                 for(i=0; i<2; i++)
2112                 {
2113                         asm volatile(
2114                                 "mov %2, %%"REG_a"              \n\t"
2115                                 "movq "MANGLE(bgr2YCoeff)", %%mm6               \n\t"
2116                                 "movq "MANGLE(w1111)", %%mm5            \n\t"
2117                                 "pxor %%mm7, %%mm7              \n\t"
2118                                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_b"\n\t"
2119                                 ".balign 16                     \n\t"
2120                                 "1:                             \n\t"
2121                                 PREFETCH" 64(%0, %%"REG_b")     \n\t"
2122                                 "movd (%0, %%"REG_b"), %%mm0    \n\t"
2123                                 "movd 3(%0, %%"REG_b"), %%mm1   \n\t"
2124                                 "punpcklbw %%mm7, %%mm0         \n\t"
2125                                 "punpcklbw %%mm7, %%mm1         \n\t"
2126                                 "movd 6(%0, %%"REG_b"), %%mm2   \n\t"
2127                                 "movd 9(%0, %%"REG_b"), %%mm3   \n\t"
2128                                 "punpcklbw %%mm7, %%mm2         \n\t"
2129                                 "punpcklbw %%mm7, %%mm3         \n\t"
2130                                 "pmaddwd %%mm6, %%mm0           \n\t"
2131                                 "pmaddwd %%mm6, %%mm1           \n\t"
2132                                 "pmaddwd %%mm6, %%mm2           \n\t"
2133                                 "pmaddwd %%mm6, %%mm3           \n\t"
2134 #ifndef FAST_BGR2YV12
2135                                 "psrad $8, %%mm0                \n\t"
2136                                 "psrad $8, %%mm1                \n\t"
2137                                 "psrad $8, %%mm2                \n\t"
2138                                 "psrad $8, %%mm3                \n\t"
2139 #endif
2140                                 "packssdw %%mm1, %%mm0          \n\t"
2141                                 "packssdw %%mm3, %%mm2          \n\t"
2142                                 "pmaddwd %%mm5, %%mm0           \n\t"
2143                                 "pmaddwd %%mm5, %%mm2           \n\t"
2144                                 "packssdw %%mm2, %%mm0          \n\t"
2145                                 "psraw $7, %%mm0                \n\t"
2146
2147                                 "movd 12(%0, %%"REG_b"), %%mm4  \n\t"
2148                                 "movd 15(%0, %%"REG_b"), %%mm1  \n\t"
2149                                 "punpcklbw %%mm7, %%mm4         \n\t"
2150                                 "punpcklbw %%mm7, %%mm1         \n\t"
2151                                 "movd 18(%0, %%"REG_b"), %%mm2  \n\t"
2152                                 "movd 21(%0, %%"REG_b"), %%mm3  \n\t"
2153                                 "punpcklbw %%mm7, %%mm2         \n\t"
2154                                 "punpcklbw %%mm7, %%mm3         \n\t"
2155                                 "pmaddwd %%mm6, %%mm4           \n\t"
2156                                 "pmaddwd %%mm6, %%mm1           \n\t"
2157                                 "pmaddwd %%mm6, %%mm2           \n\t"
2158                                 "pmaddwd %%mm6, %%mm3           \n\t"
2159 #ifndef FAST_BGR2YV12
2160                                 "psrad $8, %%mm4                \n\t"
2161                                 "psrad $8, %%mm1                \n\t"
2162                                 "psrad $8, %%mm2                \n\t"
2163                                 "psrad $8, %%mm3                \n\t"
2164 #endif
2165                                 "packssdw %%mm1, %%mm4          \n\t"
2166                                 "packssdw %%mm3, %%mm2          \n\t"
2167                                 "pmaddwd %%mm5, %%mm4           \n\t"
2168                                 "pmaddwd %%mm5, %%mm2           \n\t"
2169                                 "add $24, %%"REG_b"             \n\t"
2170                                 "packssdw %%mm2, %%mm4          \n\t"
2171                                 "psraw $7, %%mm4                \n\t"
2172
2173                                 "packuswb %%mm4, %%mm0          \n\t"
2174                                 "paddusb "MANGLE(bgr2YOffset)", %%mm0   \n\t"
2175
2176                                 MOVNTQ" %%mm0, (%1, %%"REG_a")  \n\t"
2177                                 "add $8, %%"REG_a"              \n\t"
2178                                 " js 1b                         \n\t"
2179                                 : : "r" (src+width*3), "r" (ydst+width), "g" (-width)
2180                                 : "%"REG_a, "%"REG_b
2181                         );
2182                         ydst += lumStride;
2183                         src  += srcStride;
2184                 }
2185                 src -= srcStride*2;
2186                 asm volatile(
2187                         "mov %4, %%"REG_a"              \n\t"
2188                         "movq "MANGLE(w1111)", %%mm5            \n\t"
2189                         "movq "MANGLE(bgr2UCoeff)", %%mm6               \n\t"
2190                         "pxor %%mm7, %%mm7              \n\t"
2191                         "lea (%%"REG_a", %%"REG_a", 2), %%"REG_b"\n\t"
2192                         "add %%"REG_b", %%"REG_b"       \n\t"
2193                         ".balign 16                     \n\t"
2194                         "1:                             \n\t"
2195                         PREFETCH" 64(%0, %%"REG_b")     \n\t"
2196                         PREFETCH" 64(%1, %%"REG_b")     \n\t"
2197 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2198                         "movq (%0, %%"REG_b"), %%mm0    \n\t"
2199                         "movq (%1, %%"REG_b"), %%mm1    \n\t"
2200                         "movq 6(%0, %%"REG_b"), %%mm2   \n\t"
2201                         "movq 6(%1, %%"REG_b"), %%mm3   \n\t"
2202                         PAVGB" %%mm1, %%mm0             \n\t"
2203                         PAVGB" %%mm3, %%mm2             \n\t"
2204                         "movq %%mm0, %%mm1              \n\t"
2205                         "movq %%mm2, %%mm3              \n\t"
2206                         "psrlq $24, %%mm0               \n\t"
2207                         "psrlq $24, %%mm2               \n\t"
2208                         PAVGB" %%mm1, %%mm0             \n\t"
2209                         PAVGB" %%mm3, %%mm2             \n\t"
2210                         "punpcklbw %%mm7, %%mm0         \n\t"
2211                         "punpcklbw %%mm7, %%mm2         \n\t"
2212 #else
2213                         "movd (%0, %%"REG_b"), %%mm0    \n\t"
2214                         "movd (%1, %%"REG_b"), %%mm1    \n\t"
2215                         "movd 3(%0, %%"REG_b"), %%mm2   \n\t"
2216                         "movd 3(%1, %%"REG_b"), %%mm3   \n\t"
2217                         "punpcklbw %%mm7, %%mm0         \n\t"
2218                         "punpcklbw %%mm7, %%mm1         \n\t"
2219                         "punpcklbw %%mm7, %%mm2         \n\t"
2220                         "punpcklbw %%mm7, %%mm3         \n\t"
2221                         "paddw %%mm1, %%mm0             \n\t"
2222                         "paddw %%mm3, %%mm2             \n\t"
2223                         "paddw %%mm2, %%mm0             \n\t"
2224                         "movd 6(%0, %%"REG_b"), %%mm4   \n\t"
2225                         "movd 6(%1, %%"REG_b"), %%mm1   \n\t"
2226                         "movd 9(%0, %%"REG_b"), %%mm2   \n\t"
2227                         "movd 9(%1, %%"REG_b"), %%mm3   \n\t"
2228                         "punpcklbw %%mm7, %%mm4         \n\t"
2229                         "punpcklbw %%mm7, %%mm1         \n\t"
2230                         "punpcklbw %%mm7, %%mm2         \n\t"
2231                         "punpcklbw %%mm7, %%mm3         \n\t"
2232                         "paddw %%mm1, %%mm4             \n\t"
2233                         "paddw %%mm3, %%mm2             \n\t"
2234                         "paddw %%mm4, %%mm2             \n\t"
2235                         "psrlw $2, %%mm0                \n\t"
2236                         "psrlw $2, %%mm2                \n\t"
2237 #endif
2238                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
2239                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
2240
2241                         "pmaddwd %%mm0, %%mm1           \n\t"
2242                         "pmaddwd %%mm2, %%mm3           \n\t"
2243                         "pmaddwd %%mm6, %%mm0           \n\t"
2244                         "pmaddwd %%mm6, %%mm2           \n\t"
2245 #ifndef FAST_BGR2YV12
2246                         "psrad $8, %%mm0                \n\t"
2247                         "psrad $8, %%mm1                \n\t"
2248                         "psrad $8, %%mm2                \n\t"
2249                         "psrad $8, %%mm3                \n\t"
2250 #endif
2251                         "packssdw %%mm2, %%mm0          \n\t"
2252                         "packssdw %%mm3, %%mm1          \n\t"
2253                         "pmaddwd %%mm5, %%mm0           \n\t"
2254                         "pmaddwd %%mm5, %%mm1           \n\t"
2255                         "packssdw %%mm1, %%mm0          \n\t" // V1 V0 U1 U0
2256                         "psraw $7, %%mm0                \n\t"
2257
2258 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2259                         "movq 12(%0, %%"REG_b"), %%mm4  \n\t"
2260                         "movq 12(%1, %%"REG_b"), %%mm1  \n\t"
2261                         "movq 18(%0, %%"REG_b"), %%mm2  \n\t"
2262                         "movq 18(%1, %%"REG_b"), %%mm3  \n\t"
2263                         PAVGB" %%mm1, %%mm4             \n\t"
2264                         PAVGB" %%mm3, %%mm2             \n\t"
2265                         "movq %%mm4, %%mm1              \n\t"
2266                         "movq %%mm2, %%mm3              \n\t"
2267                         "psrlq $24, %%mm4               \n\t"
2268                         "psrlq $24, %%mm2               \n\t"
2269                         PAVGB" %%mm1, %%mm4             \n\t"
2270                         PAVGB" %%mm3, %%mm2             \n\t"
2271                         "punpcklbw %%mm7, %%mm4         \n\t"
2272                         "punpcklbw %%mm7, %%mm2         \n\t"
2273 #else
2274                         "movd 12(%0, %%"REG_b"), %%mm4  \n\t"
2275                         "movd 12(%1, %%"REG_b"), %%mm1  \n\t"
2276                         "movd 15(%0, %%"REG_b"), %%mm2  \n\t"
2277                         "movd 15(%1, %%"REG_b"), %%mm3  \n\t"
2278                         "punpcklbw %%mm7, %%mm4         \n\t"
2279                         "punpcklbw %%mm7, %%mm1         \n\t"
2280                         "punpcklbw %%mm7, %%mm2         \n\t"
2281                         "punpcklbw %%mm7, %%mm3         \n\t"
2282                         "paddw %%mm1, %%mm4             \n\t"
2283                         "paddw %%mm3, %%mm2             \n\t"
2284                         "paddw %%mm2, %%mm4             \n\t"
2285                         "movd 18(%0, %%"REG_b"), %%mm5  \n\t"
2286                         "movd 18(%1, %%"REG_b"), %%mm1  \n\t"
2287                         "movd 21(%0, %%"REG_b"), %%mm2  \n\t"
2288                         "movd 21(%1, %%"REG_b"), %%mm3  \n\t"
2289                         "punpcklbw %%mm7, %%mm5         \n\t"
2290                         "punpcklbw %%mm7, %%mm1         \n\t"
2291                         "punpcklbw %%mm7, %%mm2         \n\t"
2292                         "punpcklbw %%mm7, %%mm3         \n\t"
2293                         "paddw %%mm1, %%mm5             \n\t"
2294                         "paddw %%mm3, %%mm2             \n\t"
2295                         "paddw %%mm5, %%mm2             \n\t"
2296                         "movq "MANGLE(w1111)", %%mm5            \n\t"
2297                         "psrlw $2, %%mm4                \n\t"
2298                         "psrlw $2, %%mm2                \n\t"
2299 #endif
2300                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
2301                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
2302
2303                         "pmaddwd %%mm4, %%mm1           \n\t"
2304                         "pmaddwd %%mm2, %%mm3           \n\t"
2305                         "pmaddwd %%mm6, %%mm4           \n\t"
2306                         "pmaddwd %%mm6, %%mm2           \n\t"
2307 #ifndef FAST_BGR2YV12
2308                         "psrad $8, %%mm4                \n\t"
2309                         "psrad $8, %%mm1                \n\t"
2310                         "psrad $8, %%mm2                \n\t"
2311                         "psrad $8, %%mm3                \n\t"
2312 #endif
2313                         "packssdw %%mm2, %%mm4          \n\t"
2314                         "packssdw %%mm3, %%mm1          \n\t"
2315                         "pmaddwd %%mm5, %%mm4           \n\t"
2316                         "pmaddwd %%mm5, %%mm1           \n\t"
2317                         "add $24, %%"REG_b"             \n\t"
2318                         "packssdw %%mm1, %%mm4          \n\t" // V3 V2 U3 U2
2319                         "psraw $7, %%mm4                \n\t"
2320
2321                         "movq %%mm0, %%mm1              \n\t"
2322                         "punpckldq %%mm4, %%mm0         \n\t"
2323                         "punpckhdq %%mm4, %%mm1         \n\t"
2324                         "packsswb %%mm1, %%mm0          \n\t"
2325                         "paddb "MANGLE(bgr2UVOffset)", %%mm0    \n\t"
2326                         "movd %%mm0, (%2, %%"REG_a")    \n\t"
2327                         "punpckhdq %%mm0, %%mm0         \n\t"
2328                         "movd %%mm0, (%3, %%"REG_a")    \n\t"
2329                         "add $4, %%"REG_a"              \n\t"
2330                         " js 1b                         \n\t"
2331                         : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth)
2332                         : "%"REG_a, "%"REG_b
2333                 );
2334
2335                 udst += chromStride;
2336                 vdst += chromStride;
2337                 src  += srcStride*2;
2338         }
2339
2340         asm volatile(   EMMS" \n\t"
2341                         SFENCE" \n\t"
2342                         :::"memory");
2343 #else
2344         y=0;
2345 #endif
2346         for(; y<height; y+=2)
2347         {
2348                 long i;
2349                 for(i=0; i<chromWidth; i++)
2350                 {
2351                         unsigned int b= src[6*i+0];
2352                         unsigned int g= src[6*i+1];
2353                         unsigned int r= src[6*i+2];
2354
2355                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2356                         unsigned int V  =  ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128;
2357                         unsigned int U  =  ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128;
2358
2359                         udst[i]         = U;
2360                         vdst[i]         = V;
2361                         ydst[2*i]       = Y;
2362
2363                         b= src[6*i+3];
2364                         g= src[6*i+4];
2365                         r= src[6*i+5];
2366
2367                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2368                         ydst[2*i+1]     = Y;
2369                 }
2370                 ydst += lumStride;
2371                 src  += srcStride;
2372
2373                 for(i=0; i<chromWidth; i++)
2374                 {
2375                         unsigned int b= src[6*i+0];
2376                         unsigned int g= src[6*i+1];
2377                         unsigned int r= src[6*i+2];
2378
2379                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2380
2381                         ydst[2*i]       = Y;
2382
2383                         b= src[6*i+3];
2384                         g= src[6*i+4];
2385                         r= src[6*i+5];
2386
2387                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2388                         ydst[2*i+1]     = Y;
2389                 }
2390                 udst += chromStride;
2391                 vdst += chromStride;
2392                 ydst += lumStride;
2393                 src  += srcStride;
2394         }
2395 }
2396
2397 void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest,
2398                             long width, long height, long src1Stride,
2399                             long src2Stride, long dstStride){
2400         long h;
2401
2402         for(h=0; h < height; h++)
2403         {
2404                 long w;
2405
2406 #ifdef HAVE_MMX
2407 #ifdef HAVE_SSE2
2408                 asm(
2409                         "xor %%"REG_a", %%"REG_a"       \n\t"
2410                         "1:                             \n\t"
2411                         PREFETCH" 64(%1, %%"REG_a")     \n\t"
2412                         PREFETCH" 64(%2, %%"REG_a")     \n\t"
2413                         "movdqa (%1, %%"REG_a"), %%xmm0 \n\t"
2414                         "movdqa (%1, %%"REG_a"), %%xmm1 \n\t"
2415                         "movdqa (%2, %%"REG_a"), %%xmm2 \n\t"
2416                         "punpcklbw %%xmm2, %%xmm0       \n\t"
2417                         "punpckhbw %%xmm2, %%xmm1       \n\t"
2418                         "movntdq %%xmm0, (%0, %%"REG_a", 2)\n\t"
2419                         "movntdq %%xmm1, 16(%0, %%"REG_a", 2)\n\t"
2420                         "add $16, %%"REG_a"             \n\t"
2421                         "cmp %3, %%"REG_a"              \n\t"
2422                         " jb 1b                         \n\t"
2423                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2424                         : "memory", "%"REG_a""
2425                 );
2426 #else
2427                 asm(
2428                         "xor %%"REG_a", %%"REG_a"       \n\t"
2429                         "1:                             \n\t"
2430                         PREFETCH" 64(%1, %%"REG_a")     \n\t"
2431                         PREFETCH" 64(%2, %%"REG_a")     \n\t"
2432                         "movq (%1, %%"REG_a"), %%mm0    \n\t"
2433                         "movq 8(%1, %%"REG_a"), %%mm2   \n\t"
2434                         "movq %%mm0, %%mm1              \n\t"
2435                         "movq %%mm2, %%mm3              \n\t"
2436                         "movq (%2, %%"REG_a"), %%mm4    \n\t"
2437                         "movq 8(%2, %%"REG_a"), %%mm5   \n\t"
2438                         "punpcklbw %%mm4, %%mm0         \n\t"
2439                         "punpckhbw %%mm4, %%mm1         \n\t"
2440                         "punpcklbw %%mm5, %%mm2         \n\t"
2441                         "punpckhbw %%mm5, %%mm3         \n\t"
2442                         MOVNTQ" %%mm0, (%0, %%"REG_a", 2)\n\t"
2443                         MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2)\n\t"
2444                         MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2)\n\t"
2445                         MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2)\n\t"
2446                         "add $16, %%"REG_a"             \n\t"
2447                         "cmp %3, %%"REG_a"              \n\t"
2448                         " jb 1b                         \n\t"
2449                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2450                         : "memory", "%"REG_a
2451                 );
2452 #endif
2453                 for(w= (width&(~15)); w < width; w++)
2454                 {
2455                         dest[2*w+0] = src1[w];
2456                         dest[2*w+1] = src2[w];
2457                 }
2458 #else
2459                 for(w=0; w < width; w++)
2460                 {
2461                         dest[2*w+0] = src1[w];
2462                         dest[2*w+1] = src2[w];
2463                 }
2464 #endif
2465                 dest += dstStride;
2466                 src1 += src1Stride;
2467                 src2 += src2Stride;
2468         }
2469 #ifdef HAVE_MMX
2470         asm(
2471                 EMMS" \n\t"
2472                 SFENCE" \n\t"
2473                 ::: "memory"
2474                 );
2475 #endif
2476 }
2477
2478 static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
2479                         uint8_t *dst1, uint8_t *dst2,
2480                         long width, long height,
2481                         long srcStride1, long srcStride2,
2482                         long dstStride1, long dstStride2)
2483 {
2484     long y,x,w,h;
2485     w=width/2; h=height/2;
2486 #ifdef HAVE_MMX
2487     asm volatile(
2488         PREFETCH" %0\n\t"
2489         PREFETCH" %1\n\t"
2490         ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
2491 #endif
2492     for(y=0;y<h;y++){
2493         const uint8_t* s1=src1+srcStride1*(y>>1);
2494         uint8_t* d=dst1+dstStride1*y;
2495         x=0;
2496 #ifdef HAVE_MMX
2497         for(;x<w-31;x+=32)
2498         {
2499             asm volatile(
2500                 PREFETCH" 32%1\n\t"
2501                 "movq   %1, %%mm0\n\t"
2502                 "movq   8%1, %%mm2\n\t"
2503                 "movq   16%1, %%mm4\n\t"
2504                 "movq   24%1, %%mm6\n\t"
2505                 "movq   %%mm0, %%mm1\n\t"
2506                 "movq   %%mm2, %%mm3\n\t"
2507                 "movq   %%mm4, %%mm5\n\t"
2508                 "movq   %%mm6, %%mm7\n\t"
2509                 "punpcklbw %%mm0, %%mm0\n\t"
2510                 "punpckhbw %%mm1, %%mm1\n\t"
2511                 "punpcklbw %%mm2, %%mm2\n\t"
2512                 "punpckhbw %%mm3, %%mm3\n\t"
2513                 "punpcklbw %%mm4, %%mm4\n\t"
2514                 "punpckhbw %%mm5, %%mm5\n\t"
2515                 "punpcklbw %%mm6, %%mm6\n\t"
2516                 "punpckhbw %%mm7, %%mm7\n\t"
2517                 MOVNTQ" %%mm0, %0\n\t"
2518                 MOVNTQ" %%mm1, 8%0\n\t"
2519                 MOVNTQ" %%mm2, 16%0\n\t"
2520                 MOVNTQ" %%mm3, 24%0\n\t"
2521                 MOVNTQ" %%mm4, 32%0\n\t"
2522                 MOVNTQ" %%mm5, 40%0\n\t"
2523                 MOVNTQ" %%mm6, 48%0\n\t"
2524                 MOVNTQ" %%mm7, 56%0"
2525                 :"=m"(d[2*x])
2526                 :"m"(s1[x])
2527                 :"memory");
2528         }
2529 #endif
2530         for(;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
2531     }
2532     for(y=0;y<h;y++){
2533         const uint8_t* s2=src2+srcStride2*(y>>1);
2534         uint8_t* d=dst2+dstStride2*y;
2535         x=0;
2536 #ifdef HAVE_MMX
2537         for(;x<w-31;x+=32)
2538         {
2539             asm volatile(
2540                 PREFETCH" 32%1\n\t"
2541                 "movq   %1, %%mm0\n\t"
2542                 "movq   8%1, %%mm2\n\t"
2543                 "movq   16%1, %%mm4\n\t"
2544                 "movq   24%1, %%mm6\n\t"
2545                 "movq   %%mm0, %%mm1\n\t"
2546                 "movq   %%mm2, %%mm3\n\t"
2547                 "movq   %%mm4, %%mm5\n\t"
2548                 "movq   %%mm6, %%mm7\n\t"
2549                 "punpcklbw %%mm0, %%mm0\n\t"
2550                 "punpckhbw %%mm1, %%mm1\n\t"
2551                 "punpcklbw %%mm2, %%mm2\n\t"
2552                 "punpckhbw %%mm3, %%mm3\n\t"
2553                 "punpcklbw %%mm4, %%mm4\n\t"
2554                 "punpckhbw %%mm5, %%mm5\n\t"
2555                 "punpcklbw %%mm6, %%mm6\n\t"
2556                 "punpckhbw %%mm7, %%mm7\n\t"
2557                 MOVNTQ" %%mm0, %0\n\t"
2558                 MOVNTQ" %%mm1, 8%0\n\t"
2559                 MOVNTQ" %%mm2, 16%0\n\t"
2560                 MOVNTQ" %%mm3, 24%0\n\t"
2561                 MOVNTQ" %%mm4, 32%0\n\t"
2562                 MOVNTQ" %%mm5, 40%0\n\t"
2563                 MOVNTQ" %%mm6, 48%0\n\t"
2564                 MOVNTQ" %%mm7, 56%0"
2565                 :"=m"(d[2*x])
2566                 :"m"(s2[x])
2567                 :"memory");
2568         }
2569 #endif
2570         for(;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
2571     }
2572 #ifdef HAVE_MMX
2573         asm(
2574                 EMMS" \n\t"
2575                 SFENCE" \n\t"
2576                 ::: "memory"
2577                 );
2578 #endif
2579 }
2580
2581 static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
2582                         uint8_t *dst,
2583                         long width, long height,
2584                         long srcStride1, long srcStride2,
2585                         long srcStride3, long dstStride)
2586 {
2587     long y,x,w,h;
2588     w=width/2; h=height;
2589     for(y=0;y<h;y++){
2590         const uint8_t* yp=src1+srcStride1*y;
2591         const uint8_t* up=src2+srcStride2*(y>>2);
2592         const uint8_t* vp=src3+srcStride3*(y>>2);
2593         uint8_t* d=dst+dstStride*y;
2594         x=0;
2595 #ifdef HAVE_MMX
2596         for(;x<w-7;x+=8)
2597         {
2598             asm volatile(
2599                 PREFETCH" 32(%1, %0)\n\t"
2600                 PREFETCH" 32(%2, %0)\n\t"
2601                 PREFETCH" 32(%3, %0)\n\t"
2602                 "movq   (%1, %0, 4), %%mm0\n\t"       /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2603                 "movq   (%2, %0), %%mm1\n\t"       /* U0U1U2U3U4U5U6U7 */
2604                 "movq   (%3, %0), %%mm2\n\t"         /* V0V1V2V3V4V5V6V7 */
2605                 "movq   %%mm0, %%mm3\n\t"    /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2606                 "movq   %%mm1, %%mm4\n\t"    /* U0U1U2U3U4U5U6U7 */
2607                 "movq   %%mm2, %%mm5\n\t"    /* V0V1V2V3V4V5V6V7 */
2608                 "punpcklbw %%mm1, %%mm1\n\t" /* U0U0 U1U1 U2U2 U3U3 */
2609                 "punpcklbw %%mm2, %%mm2\n\t" /* V0V0 V1V1 V2V2 V3V3 */
2610                 "punpckhbw %%mm4, %%mm4\n\t" /* U4U4 U5U5 U6U6 U7U7 */
2611                 "punpckhbw %%mm5, %%mm5\n\t" /* V4V4 V5V5 V6V6 V7V7 */
2612
2613                 "movq   %%mm1, %%mm6\n\t"
2614                 "punpcklbw %%mm2, %%mm1\n\t" /* U0V0 U0V0 U1V1 U1V1*/
2615                 "punpcklbw %%mm1, %%mm0\n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
2616                 "punpckhbw %%mm1, %%mm3\n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
2617                 MOVNTQ" %%mm0, (%4, %0, 8)\n\t"
2618                 MOVNTQ" %%mm3, 8(%4, %0, 8)\n\t"
2619                 
2620                 "punpckhbw %%mm2, %%mm6\n\t" /* U2V2 U2V2 U3V3 U3V3*/
2621                 "movq   8(%1, %0, 4), %%mm0\n\t"
2622                 "movq   %%mm0, %%mm3\n\t"
2623                 "punpcklbw %%mm6, %%mm0\n\t" /* Y U2 Y V2 Y U2 Y V2*/
2624                 "punpckhbw %%mm6, %%mm3\n\t" /* Y U3 Y V3 Y U3 Y V3*/
2625                 MOVNTQ" %%mm0, 16(%4, %0, 8)\n\t"
2626                 MOVNTQ" %%mm3, 24(%4, %0, 8)\n\t"
2627
2628                 "movq   %%mm4, %%mm6\n\t"
2629                 "movq   16(%1, %0, 4), %%mm0\n\t"
2630                 "movq   %%mm0, %%mm3\n\t"
2631                 "punpcklbw %%mm5, %%mm4\n\t"
2632                 "punpcklbw %%mm4, %%mm0\n\t" /* Y U4 Y V4 Y U4 Y V4*/
2633                 "punpckhbw %%mm4, %%mm3\n\t" /* Y U5 Y V5 Y U5 Y V5*/
2634                 MOVNTQ" %%mm0, 32(%4, %0, 8)\n\t"
2635                 MOVNTQ" %%mm3, 40(%4, %0, 8)\n\t"
2636                 
2637                 "punpckhbw %%mm5, %%mm6\n\t"
2638                 "movq   24(%1, %0, 4), %%mm0\n\t"
2639                 "movq   %%mm0, %%mm3\n\t"
2640                 "punpcklbw %%mm6, %%mm0\n\t" /* Y U6 Y V6 Y U6 Y V6*/
2641                 "punpckhbw %%mm6, %%mm3\n\t" /* Y U7 Y V7 Y U7 Y V7*/
2642                 MOVNTQ" %%mm0, 48(%4, %0, 8)\n\t"
2643                 MOVNTQ" %%mm3, 56(%4, %0, 8)\n\t"
2644
2645                 : "+r" (x)
2646                 : "r"(yp), "r" (up), "r"(vp), "r"(d)
2647                 :"memory");
2648         }
2649 #endif
2650         for(; x<w; x++)
2651         {
2652             const long x2= x<<2;
2653             d[8*x+0]=yp[x2];
2654             d[8*x+1]=up[x];
2655             d[8*x+2]=yp[x2+1];
2656             d[8*x+3]=vp[x];
2657             d[8*x+4]=yp[x2+2];
2658             d[8*x+5]=up[x];
2659             d[8*x+6]=yp[x2+3];
2660             d[8*x+7]=vp[x];
2661         }
2662     }
2663 #ifdef HAVE_MMX
2664         asm(
2665                 EMMS" \n\t"
2666                 SFENCE" \n\t"
2667                 ::: "memory"
2668                 );
2669 #endif
2670 }