put all integer init code to compute n^(4/3) - memory alloc and header fixes
[ffmpeg.git] / postproc / rgb2rgb_template.c
1 /*
2  *
3  *  rgb2rgb.c, Software RGB to RGB convertor
4  *  pluralize by Software PAL8 to RGB convertor
5  *               Software YUV to YUV convertor
6  *               Software YUV to RGB convertor
7  *  Written by Nick Kurshev.
8  *  palette & yuv & runtime cpu stuff by Michael (michaelni@gmx.at) (under GPL)
9  */
10
11 #undef PREFETCH
12 #undef MOVNTQ
13 #undef EMMS
14 #undef SFENCE
15 #undef MMREG_SIZE
16 #undef PREFETCHW
17 #undef PAVGB
18
19 #ifdef HAVE_SSE2
20 #define MMREG_SIZE 16
21 #else
22 #define MMREG_SIZE 8
23 #endif
24
25 #ifdef HAVE_3DNOW
26 #define PREFETCH  "prefetch"
27 #define PREFETCHW "prefetchw"
28 #define PAVGB     "pavgusb"
29 #elif defined ( HAVE_MMX2 )
30 #define PREFETCH "prefetchnta"
31 #define PREFETCHW "prefetcht0"
32 #define PAVGB     "pavgb"
33 #else
34 #define PREFETCH "/nop"
35 #define PREFETCHW "/nop"
36 #endif
37
38 #ifdef HAVE_3DNOW
39 /* On K6 femms is faster of emms. On K7 femms is directly mapped on emms. */
40 #define EMMS     "femms"
41 #else
42 #define EMMS     "emms"
43 #endif
44
45 #ifdef HAVE_MMX2
46 #define MOVNTQ "movntq"
47 #define SFENCE "sfence"
48 #else
49 #define MOVNTQ "movq"
50 #define SFENCE "/nop"
51 #endif
52
53 static inline void RENAME(rgb24to32)(const uint8_t *src,uint8_t *dst,unsigned src_size)
54 {
55   uint8_t *dest = dst;
56   const uint8_t *s = src;
57   const uint8_t *end;
58 #ifdef HAVE_MMX
59   const uint8_t *mm_end;
60 #endif
61   end = s + src_size;
62 #ifdef HAVE_MMX
63   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
64   mm_end = end - 23;
65   __asm __volatile("movq        %0, %%mm7"::"m"(mask32):"memory");
66   while(s < mm_end)
67   {
68     __asm __volatile(
69         PREFETCH"       32%1\n\t"
70         "movd   %1, %%mm0\n\t"
71         "punpckldq 3%1, %%mm0\n\t"
72         "movd   6%1, %%mm1\n\t"
73         "punpckldq 9%1, %%mm1\n\t"
74         "movd   12%1, %%mm2\n\t"
75         "punpckldq 15%1, %%mm2\n\t"
76         "movd   18%1, %%mm3\n\t"
77         "punpckldq 21%1, %%mm3\n\t"
78         "pand   %%mm7, %%mm0\n\t"
79         "pand   %%mm7, %%mm1\n\t"
80         "pand   %%mm7, %%mm2\n\t"
81         "pand   %%mm7, %%mm3\n\t"
82         MOVNTQ" %%mm0, %0\n\t"
83         MOVNTQ" %%mm1, 8%0\n\t"
84         MOVNTQ" %%mm2, 16%0\n\t"
85         MOVNTQ" %%mm3, 24%0"
86         :"=m"(*dest)
87         :"m"(*s)
88         :"memory");
89     dest += 32;
90     s += 24;
91   }
92   __asm __volatile(SFENCE:::"memory");
93   __asm __volatile(EMMS:::"memory");
94 #endif
95   while(s < end)
96   {
97     *dest++ = *s++;
98     *dest++ = *s++;
99     *dest++ = *s++;
100     *dest++ = 0;
101   }
102 }
103
104 static inline void RENAME(rgb32to24)(const uint8_t *src,uint8_t *dst,unsigned src_size)
105 {
106   uint8_t *dest = dst;
107   const uint8_t *s = src;
108   const uint8_t *end;
109 #ifdef HAVE_MMX
110   const uint8_t *mm_end;
111 #endif
112   end = s + src_size;
113 #ifdef HAVE_MMX
114   __asm __volatile(PREFETCH"    %0"::"m"(*s):"memory");
115   mm_end = end - 31;
116   while(s < mm_end)
117   {
118     __asm __volatile(
119         PREFETCH"       32%1\n\t"
120         "movq   %1, %%mm0\n\t"
121         "movq   8%1, %%mm1\n\t"
122         "movq   16%1, %%mm4\n\t"
123         "movq   24%1, %%mm5\n\t"
124         "movq   %%mm0, %%mm2\n\t"
125         "movq   %%mm1, %%mm3\n\t"
126         "movq   %%mm4, %%mm6\n\t"
127         "movq   %%mm5, %%mm7\n\t"
128         "psrlq  $8, %%mm2\n\t"
129         "psrlq  $8, %%mm3\n\t"
130         "psrlq  $8, %%mm6\n\t"
131         "psrlq  $8, %%mm7\n\t"
132         "pand   %2, %%mm0\n\t"
133         "pand   %2, %%mm1\n\t"
134         "pand   %2, %%mm4\n\t"
135         "pand   %2, %%mm5\n\t"
136         "pand   %3, %%mm2\n\t"
137         "pand   %3, %%mm3\n\t"
138         "pand   %3, %%mm6\n\t"
139         "pand   %3, %%mm7\n\t"
140         "por    %%mm2, %%mm0\n\t"
141         "por    %%mm3, %%mm1\n\t"
142         "por    %%mm6, %%mm4\n\t"
143         "por    %%mm7, %%mm5\n\t"
144
145         "movq   %%mm1, %%mm2\n\t"
146         "movq   %%mm4, %%mm3\n\t"
147         "psllq  $48, %%mm2\n\t"
148         "psllq  $32, %%mm3\n\t"
149         "pand   %4, %%mm2\n\t"
150         "pand   %5, %%mm3\n\t"
151         "por    %%mm2, %%mm0\n\t"
152         "psrlq  $16, %%mm1\n\t"
153         "psrlq  $32, %%mm4\n\t"
154         "psllq  $16, %%mm5\n\t"
155         "por    %%mm3, %%mm1\n\t"
156         "pand   %6, %%mm5\n\t"
157         "por    %%mm5, %%mm4\n\t"
158
159         MOVNTQ" %%mm0, %0\n\t"
160         MOVNTQ" %%mm1, 8%0\n\t"
161         MOVNTQ" %%mm4, 16%0"
162         :"=m"(*dest)
163         :"m"(*s),"m"(mask24l),
164          "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
165         :"memory");
166     dest += 24;
167     s += 32;
168   }
169   __asm __volatile(SFENCE:::"memory");
170   __asm __volatile(EMMS:::"memory");
171 #endif
172   while(s < end)
173   {
174     *dest++ = *s++;
175     *dest++ = *s++;
176     *dest++ = *s++;
177     s++;
178   }
179 }
180
181 /*
182  Original by Strepto/Astral
183  ported to gcc & bugfixed : A'rpi
184  MMX2, 3DNOW optimization by Nick Kurshev
185  32bit c version, and and&add trick by Michael Niedermayer
186 */
187 static inline void RENAME(rgb15to16)(const uint8_t *src,uint8_t *dst,unsigned src_size)
188 {
189 #ifdef HAVE_MMX
190   register int offs=15-src_size;
191   register const char* s=src-offs;
192   register char* d=dst-offs;
193   __asm __volatile(PREFETCH"    %0"::"m"(*(s+offs)));
194   __asm __volatile(
195         "movq   %0, %%mm4\n\t"
196         ::"m"(mask15s));
197   while(offs<0)
198   {
199         __asm __volatile(
200                 PREFETCH"       32%1\n\t"
201                 "movq   %1, %%mm0\n\t"
202                 "movq   8%1, %%mm2\n\t"
203                 "movq   %%mm0, %%mm1\n\t"
204                 "movq   %%mm2, %%mm3\n\t"
205                 "pand   %%mm4, %%mm0\n\t"
206                 "pand   %%mm4, %%mm2\n\t"
207                 "paddw  %%mm1, %%mm0\n\t"
208                 "paddw  %%mm3, %%mm2\n\t"
209                 MOVNTQ" %%mm0, %0\n\t"
210                 MOVNTQ" %%mm2, 8%0"
211                 :"=m"(*(d+offs))
212                 :"m"(*(s+offs))
213                 );
214         offs+=16;
215   }
216   __asm __volatile(SFENCE:::"memory");
217   __asm __volatile(EMMS:::"memory");
218 #else
219 #if 0
220    const uint16_t *s1=( uint16_t * )src;
221    uint16_t *d1=( uint16_t * )dst;
222    uint16_t *e=((uint8_t *)s1)+src_size;
223    while( s1<e ){
224      register int x=*( s1++ );
225      /* rrrrrggggggbbbbb
226         0rrrrrgggggbbbbb
227         0111 1111 1110 0000=0x7FE0
228         00000000000001 1111=0x001F */
229      *( d1++ )=( x&0x001F )|( ( x&0x7FE0 )<<1 );
230    }
231 #else
232         const unsigned *s1=( unsigned * )src;
233         unsigned *d1=( unsigned * )dst;
234         int i;
235         int size= src_size>>2;
236         for(i=0; i<size; i++)
237         {
238                 register int x= s1[i];
239 //              d1[i] = x + (x&0x7FE07FE0); //faster but need msbit =0 which might not allways be true
240                 d1[i] = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
241
242         }
243 #endif
244 #endif
245 }
246
247 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, unsigned src_size)
248 {
249 #ifdef HAVE_MMX
250         const uint8_t *s = src;
251         const uint8_t *end,*mm_end;
252         uint16_t *d = (uint16_t *)dst;
253         end = s + src_size;
254         mm_end = end - 15;
255         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
256         __asm __volatile(
257             "movq       %0, %%mm7\n\t"
258             "movq       %1, %%mm6\n\t"
259             ::"m"(red_16mask),"m"(green_16mask));
260         while(s < mm_end)
261         {
262             __asm __volatile(
263                 PREFETCH" 32%1\n\t"
264                 "movd   %1, %%mm0\n\t"
265                 "movd   4%1, %%mm3\n\t"
266                 "punpckldq 8%1, %%mm0\n\t"
267                 "punpckldq 12%1, %%mm3\n\t"
268                 "movq   %%mm0, %%mm1\n\t"
269                 "movq   %%mm0, %%mm2\n\t"
270                 "movq   %%mm3, %%mm4\n\t"
271                 "movq   %%mm3, %%mm5\n\t"
272                 "psrlq  $3, %%mm0\n\t"
273                 "psrlq  $3, %%mm3\n\t"
274                 "pand   %2, %%mm0\n\t"
275                 "pand   %2, %%mm3\n\t"
276                 "psrlq  $5, %%mm1\n\t"
277                 "psrlq  $5, %%mm4\n\t"
278                 "pand   %%mm6, %%mm1\n\t"
279                 "pand   %%mm6, %%mm4\n\t"
280                 "psrlq  $8, %%mm2\n\t"
281                 "psrlq  $8, %%mm5\n\t"
282                 "pand   %%mm7, %%mm2\n\t"
283                 "pand   %%mm7, %%mm5\n\t"
284                 "por    %%mm1, %%mm0\n\t"
285                 "por    %%mm4, %%mm3\n\t"
286                 "por    %%mm2, %%mm0\n\t"
287                 "por    %%mm5, %%mm3\n\t"
288                 "psllq  $16, %%mm3\n\t"
289                 "por    %%mm3, %%mm0\n\t"
290                 MOVNTQ" %%mm0, %0\n\t"
291                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
292                 d += 4;
293                 s += 16;
294         }
295         while(s < end)
296         {
297                 const int b= *s++;
298                 const int g= *s++;
299                 const int r= *s++;
300                 s++;
301                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
302         }
303         __asm __volatile(SFENCE:::"memory");
304         __asm __volatile(EMMS:::"memory");
305 #else
306         unsigned j,i,num_pixels=src_size/4;
307         uint16_t *d = (uint16_t *)dst;
308         for(i=0,j=0; j<num_pixels; i+=4,j++)
309         {
310                 const int b= src[i+0];
311                 const int g= src[i+1];
312                 const int r= src[i+2];
313
314                 d[j]= (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
315         }
316 #endif
317 }
318
319 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, unsigned src_size)
320 {
321 #ifdef HAVE_MMX
322         const uint8_t *s = src;
323         const uint8_t *end,*mm_end;
324         uint16_t *d = (uint16_t *)dst;
325         end = s + src_size;
326         mm_end = end - 15;
327         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
328         __asm __volatile(
329             "movq       %0, %%mm7\n\t"
330             "movq       %1, %%mm6\n\t"
331             ::"m"(red_15mask),"m"(green_15mask));
332         while(s < mm_end)
333         {
334             __asm __volatile(
335                 PREFETCH" 32%1\n\t"
336                 "movd   %1, %%mm0\n\t"
337                 "movd   4%1, %%mm3\n\t"
338                 "punpckldq 8%1, %%mm0\n\t"
339                 "punpckldq 12%1, %%mm3\n\t"
340                 "movq   %%mm0, %%mm1\n\t"
341                 "movq   %%mm0, %%mm2\n\t"
342                 "movq   %%mm3, %%mm4\n\t"
343                 "movq   %%mm3, %%mm5\n\t"
344                 "psrlq  $3, %%mm0\n\t"
345                 "psrlq  $3, %%mm3\n\t"
346                 "pand   %2, %%mm0\n\t"
347                 "pand   %2, %%mm3\n\t"
348                 "psrlq  $6, %%mm1\n\t"
349                 "psrlq  $6, %%mm4\n\t"
350                 "pand   %%mm6, %%mm1\n\t"
351                 "pand   %%mm6, %%mm4\n\t"
352                 "psrlq  $9, %%mm2\n\t"
353                 "psrlq  $9, %%mm5\n\t"
354                 "pand   %%mm7, %%mm2\n\t"
355                 "pand   %%mm7, %%mm5\n\t"
356                 "por    %%mm1, %%mm0\n\t"
357                 "por    %%mm4, %%mm3\n\t"
358                 "por    %%mm2, %%mm0\n\t"
359                 "por    %%mm5, %%mm3\n\t"
360                 "psllq  $16, %%mm3\n\t"
361                 "por    %%mm3, %%mm0\n\t"
362                 MOVNTQ" %%mm0, %0\n\t"
363                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
364                 d += 4;
365                 s += 16;
366         }
367         while(s < end)
368         {
369                 const int b= *s++;
370                 const int g= *s++;
371                 const int r= *s++;
372                 s++;
373                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
374         }
375         __asm __volatile(SFENCE:::"memory");
376         __asm __volatile(EMMS:::"memory");
377 #else
378         unsigned j,i,num_pixels=src_size/4;
379         uint16_t *d = (uint16_t *)dst;
380         for(i=0,j=0; j<num_pixels; i+=4,j++)
381         {
382                 const int b= src[i+0];
383                 const int g= src[i+1];
384                 const int r= src[i+2];
385
386                 d[j]= (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
387         }
388 #endif
389 }
390
391 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, unsigned src_size)
392 {
393 #ifdef HAVE_MMX
394         const uint8_t *s = src;
395         const uint8_t *end,*mm_end;
396         uint16_t *d = (uint16_t *)dst;
397         end = s + src_size;
398         mm_end = end - 11;
399         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
400         __asm __volatile(
401             "movq       %0, %%mm7\n\t"
402             "movq       %1, %%mm6\n\t"
403             ::"m"(red_16mask),"m"(green_16mask));
404         while(s < mm_end)
405         {
406             __asm __volatile(
407                 PREFETCH" 32%1\n\t"
408                 "movd   %1, %%mm0\n\t"
409                 "movd   3%1, %%mm3\n\t"
410                 "punpckldq 6%1, %%mm0\n\t"
411                 "punpckldq 9%1, %%mm3\n\t"
412                 "movq   %%mm0, %%mm1\n\t"
413                 "movq   %%mm0, %%mm2\n\t"
414                 "movq   %%mm3, %%mm4\n\t"
415                 "movq   %%mm3, %%mm5\n\t"
416                 "psrlq  $3, %%mm0\n\t"
417                 "psrlq  $3, %%mm3\n\t"
418                 "pand   %2, %%mm0\n\t"
419                 "pand   %2, %%mm3\n\t"
420                 "psrlq  $5, %%mm1\n\t"
421                 "psrlq  $5, %%mm4\n\t"
422                 "pand   %%mm6, %%mm1\n\t"
423                 "pand   %%mm6, %%mm4\n\t"
424                 "psrlq  $8, %%mm2\n\t"
425                 "psrlq  $8, %%mm5\n\t"
426                 "pand   %%mm7, %%mm2\n\t"
427                 "pand   %%mm7, %%mm5\n\t"
428                 "por    %%mm1, %%mm0\n\t"
429                 "por    %%mm4, %%mm3\n\t"
430                 "por    %%mm2, %%mm0\n\t"
431                 "por    %%mm5, %%mm3\n\t"
432                 "psllq  $16, %%mm3\n\t"
433                 "por    %%mm3, %%mm0\n\t"
434                 MOVNTQ" %%mm0, %0\n\t"
435                 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
436                 d += 4;
437                 s += 12;
438         }
439         while(s < end)
440         {
441                 const int b= *s++;
442                 const int g= *s++;
443                 const int r= *s++;
444                 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
445         }
446         __asm __volatile(SFENCE:::"memory");
447         __asm __volatile(EMMS:::"memory");
448 #else
449         unsigned j,i,num_pixels=src_size/3;
450         uint16_t *d = (uint16_t *)dst;
451         for(i=0,j=0; j<num_pixels; i+=3,j++)
452         {
453                 const int b= src[i+0];
454                 const int g= src[i+1];
455                 const int r= src[i+2];
456
457                 d[j]= (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
458         }
459 #endif
460 }
461
462 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, unsigned src_size)
463 {
464 #ifdef HAVE_MMX
465         const uint8_t *s = src;
466         const uint8_t *end,*mm_end;
467         uint16_t *d = (uint16_t *)dst;
468         end = s + src_size;
469         mm_end = end -11;
470         __asm __volatile(PREFETCH"      %0"::"m"(*src):"memory");
471         __asm __volatile(
472             "movq       %0, %%mm7\n\t"
473             "movq       %1, %%mm6\n\t"
474             ::"m"(red_15mask),"m"(green_15mask));
475         while(s < mm_end)
476         {
477             __asm __volatile(
478                 PREFETCH" 32%1\n\t"
479                 "movd   %1, %%mm0\n\t"
480                 "movd   3%1, %%mm3\n\t"
481                 "punpckldq 6%1, %%mm0\n\t"
482                 "punpckldq 9%1, %%mm3\n\t"
483                 "movq   %%mm0, %%mm1\n\t"
484                 "movq   %%mm0, %%mm2\n\t"
485                 "movq   %%mm3, %%mm4\n\t"
486                 "movq   %%mm3, %%mm5\n\t"
487                 "psrlq  $3, %%mm0\n\t"
488                 "psrlq  $3, %%mm3\n\t"
489                 "pand   %2, %%mm0\n\t"
490                 "pand   %2, %%mm3\n\t"
491                 "psrlq  $6, %%mm1\n\t"
492                 "psrlq  $6, %%mm4\n\t"
493                 "pand   %%mm6, %%mm1\n\t"
494                 "pand   %%mm6, %%mm4\n\t"
495                 "psrlq  $9, %%mm2\n\t"
496                 "psrlq  $9, %%mm5\n\t"
497                 "pand   %%mm7, %%mm2\n\t"
498                 "pand   %%mm7, %%mm5\n\t"
499                 "por    %%mm1, %%mm0\n\t"
500                 "por    %%mm4, %%mm3\n\t"
501                 "por    %%mm2, %%mm0\n\t"
502                 "por    %%mm5, %%mm3\n\t"
503                 "psllq  $16, %%mm3\n\t"
504                 "por    %%mm3, %%mm0\n\t"
505                 MOVNTQ" %%mm0, %0\n\t"
506                 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
507                 d += 4;
508                 s += 12;
509         }
510         while(s < end)
511         {
512                 const int b= *s++;
513                 const int g= *s++;
514                 const int r= *s++;
515                 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
516         }
517         __asm __volatile(SFENCE:::"memory");
518         __asm __volatile(EMMS:::"memory");
519 #else
520         unsigned j,i,num_pixels=src_size/3;
521         uint16_t *d = (uint16_t *)dst;
522         for(i=0,j=0; j<num_pixels; i+=3,j++)
523         {
524                 const int b= src[i+0];
525                 const int g= src[i+1];
526                 const int r= src[i+2];
527
528                 d[j]= (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
529         }
530 #endif
531 }
532
533 static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, unsigned int src_size)
534 {
535 #ifdef HAVE_MMX
536         asm volatile (
537                 "xorl %%eax, %%eax              \n\t"
538                 ".balign 16                     \n\t"
539                 "1:                             \n\t"
540                 PREFETCH" 32(%0, %%eax)         \n\t"
541                 "movq (%0, %%eax), %%mm0        \n\t"
542                 "movq %%mm0, %%mm1              \n\t"
543                 "movq %%mm0, %%mm2              \n\t"
544                 "pslld $16, %%mm0               \n\t"
545                 "psrld $16, %%mm1               \n\t"
546                 "pand "MANGLE(mask32r)", %%mm0          \n\t"
547                 "pand "MANGLE(mask32g)", %%mm2          \n\t"
548                 "pand "MANGLE(mask32b)", %%mm1          \n\t"
549                 "por %%mm0, %%mm2               \n\t"
550                 "por %%mm1, %%mm2               \n\t"
551                 MOVNTQ" %%mm2, (%1, %%eax)      \n\t"
552                 "addl $8, %%eax                 \n\t"
553                 "cmpl %2, %%eax                 \n\t"
554                 " jb 1b                         \n\t"
555                 :: "r" (src), "r"(dst), "r" (src_size)
556                 : "%eax"
557         );
558
559         __asm __volatile(SFENCE:::"memory");
560         __asm __volatile(EMMS:::"memory");
561 #else
562         int i;
563         int num_pixels= src_size >> 2;
564         for(i=0; i<num_pixels; i++)
565         {
566                 dst[4*i + 0] = src[4*i + 2];
567                 dst[4*i + 1] = src[4*i + 1];
568                 dst[4*i + 2] = src[4*i + 0];
569         }
570 #endif
571 }
572
573 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, unsigned int src_size)
574 {
575         int i;
576 #ifdef HAVE_MMX
577         int mmx_size= 23 - src_size;
578         asm volatile (
579                 "movq "MANGLE(mask24r)", %%mm5  \n\t"
580                 "movq "MANGLE(mask24g)", %%mm6  \n\t"
581                 "movq "MANGLE(mask24b)", %%mm7  \n\t"
582                 ".balign 16                     \n\t"
583                 "1:                             \n\t"
584                 PREFETCH" 32(%1, %%eax)         \n\t"
585                 "movq   (%1, %%eax), %%mm0      \n\t" // BGR BGR BG
586                 "movq   (%1, %%eax), %%mm1      \n\t" // BGR BGR BG
587                 "movq  2(%1, %%eax), %%mm2      \n\t" // R BGR BGR B
588                 "psllq $16, %%mm0               \n\t" // 00 BGR BGR
589                 "pand %%mm5, %%mm0              \n\t"
590                 "pand %%mm6, %%mm1              \n\t"
591                 "pand %%mm7, %%mm2              \n\t"
592                 "por %%mm0, %%mm1               \n\t"
593                 "por %%mm2, %%mm1               \n\t"                
594                 "movq  6(%1, %%eax), %%mm0      \n\t" // BGR BGR BG
595                 MOVNTQ" %%mm1,   (%2, %%eax)    \n\t" // RGB RGB RG
596                 "movq  8(%1, %%eax), %%mm1      \n\t" // R BGR BGR B
597                 "movq 10(%1, %%eax), %%mm2      \n\t" // GR BGR BGR
598                 "pand %%mm7, %%mm0              \n\t"
599                 "pand %%mm5, %%mm1              \n\t"
600                 "pand %%mm6, %%mm2              \n\t"
601                 "por %%mm0, %%mm1               \n\t"
602                 "por %%mm2, %%mm1               \n\t"                
603                 "movq 14(%1, %%eax), %%mm0      \n\t" // R BGR BGR B
604                 MOVNTQ" %%mm1,  8(%2, %%eax)    \n\t" // B RGB RGB R
605                 "movq 16(%1, %%eax), %%mm1      \n\t" // GR BGR BGR
606                 "movq 18(%1, %%eax), %%mm2      \n\t" // BGR BGR BG
607                 "pand %%mm6, %%mm0              \n\t"
608                 "pand %%mm7, %%mm1              \n\t"
609                 "pand %%mm5, %%mm2              \n\t"
610                 "por %%mm0, %%mm1               \n\t"
611                 "por %%mm2, %%mm1               \n\t"                
612                 MOVNTQ" %%mm1, 16(%2, %%eax)    \n\t"
613                 "addl $24, %%eax                \n\t"
614                 " js 1b                         \n\t"
615                 : "+a" (mmx_size)
616                 : "r" (src-mmx_size), "r"(dst-mmx_size)
617         );
618
619         __asm __volatile(SFENCE:::"memory");
620         __asm __volatile(EMMS:::"memory");
621
622         if(mmx_size==23) return; //finihsed, was multiple of 8
623         src+= src_size;
624         dst+= src_size;
625         src_size= 23 - mmx_size;
626         src-= src_size;
627         dst-= src_size;
628 #endif
629         for(i=0; i<src_size; i+=3)
630         {
631                 register int x;
632                 x          = src[i + 2];
633                 dst[i + 1] = src[i + 1];
634                 dst[i + 2] = src[i + 0];
635                 dst[i + 0] = x;
636         }
637 }
638
639 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
640         unsigned int width, unsigned int height,
641         unsigned int lumStride, unsigned int chromStride, unsigned int dstStride, int vertLumPerChroma)
642 {
643         int y;
644         const int chromWidth= width>>1;
645         for(y=0; y<height; y++)
646         {
647 #ifdef HAVE_MMX
648 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
649                 asm volatile(
650                         "xorl %%eax, %%eax              \n\t"
651                         ".balign 16                     \n\t"
652                         "1:                             \n\t"
653                         PREFETCH" 32(%1, %%eax, 2)      \n\t"
654                         PREFETCH" 32(%2, %%eax)         \n\t"
655                         PREFETCH" 32(%3, %%eax)         \n\t"
656                         "movq (%2, %%eax), %%mm0        \n\t" // U(0)
657                         "movq %%mm0, %%mm2              \n\t" // U(0)
658                         "movq (%3, %%eax), %%mm1        \n\t" // V(0)
659                         "punpcklbw %%mm1, %%mm0         \n\t" // UVUV UVUV(0)
660                         "punpckhbw %%mm1, %%mm2         \n\t" // UVUV UVUV(8)
661
662                         "movq (%1, %%eax,2), %%mm3      \n\t" // Y(0)
663                         "movq 8(%1, %%eax,2), %%mm5     \n\t" // Y(8)
664                         "movq %%mm3, %%mm4              \n\t" // Y(0)
665                         "movq %%mm5, %%mm6              \n\t" // Y(8)
666                         "punpcklbw %%mm0, %%mm3         \n\t" // YUYV YUYV(0)
667                         "punpckhbw %%mm0, %%mm4         \n\t" // YUYV YUYV(4)
668                         "punpcklbw %%mm2, %%mm5         \n\t" // YUYV YUYV(8)
669                         "punpckhbw %%mm2, %%mm6         \n\t" // YUYV YUYV(12)
670
671                         MOVNTQ" %%mm3, (%0, %%eax, 4)   \n\t"
672                         MOVNTQ" %%mm4, 8(%0, %%eax, 4)  \n\t"
673                         MOVNTQ" %%mm5, 16(%0, %%eax, 4) \n\t"
674                         MOVNTQ" %%mm6, 24(%0, %%eax, 4) \n\t"
675
676                         "addl $8, %%eax                 \n\t"
677                         "cmpl %4, %%eax                 \n\t"
678                         " jb 1b                         \n\t"
679                         ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "r" (chromWidth)
680                         : "%eax"
681                 );
682 #else
683                 int i;
684                 for(i=0; i<chromWidth; i++)
685                 {
686                         dst[4*i+0] = ysrc[2*i+0];
687                         dst[4*i+1] = usrc[i];
688                         dst[4*i+2] = ysrc[2*i+1];
689                         dst[4*i+3] = vsrc[i];
690                 }
691 #endif
692                 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
693                 {
694                         usrc += chromStride;
695                         vsrc += chromStride;
696                 }
697                 ysrc += lumStride;
698                 dst += dstStride;
699         }
700 #ifdef HAVE_MMX
701 asm(    EMMS" \n\t"
702         SFENCE" \n\t"
703         :::"memory");
704 #endif
705 }
706
707 /**
708  *
709  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
710  * problem for anyone then tell me, and ill fix it)
711  */
712 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
713         unsigned int width, unsigned int height,
714         unsigned int lumStride, unsigned int chromStride, unsigned int dstStride)
715 {
716         //FIXME interpolate chroma
717         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
718 }
719
720 /**
721  *
722  * width should be a multiple of 16
723  */
724 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
725         unsigned int width, unsigned int height,
726         unsigned int lumStride, unsigned int chromStride, unsigned int dstStride)
727 {
728         RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
729 }
730
731 /**
732  *
733  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
734  * problem for anyone then tell me, and ill fix it)
735  */
736 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
737         unsigned int width, unsigned int height,
738         unsigned int lumStride, unsigned int chromStride, unsigned int srcStride)
739 {
740         int y;
741         const int chromWidth= width>>1;
742         for(y=0; y<height; y+=2)
743         {
744 #ifdef HAVE_MMX
745                 asm volatile(
746                         "xorl %%eax, %%eax              \n\t"
747                         "pcmpeqw %%mm7, %%mm7           \n\t"
748                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
749                         ".balign 16                     \n\t"
750                         "1:                             \n\t"
751                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
752                         "movq (%0, %%eax, 4), %%mm0     \n\t" // YUYV YUYV(0)
753                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // YUYV YUYV(4)
754                         "movq %%mm0, %%mm2              \n\t" // YUYV YUYV(0)
755                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(4)
756                         "psrlw $8, %%mm0                \n\t" // U0V0 U0V0(0)
757                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(4)
758                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(0)
759                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(4)
760                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
761                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
762
763                         MOVNTQ" %%mm2, (%1, %%eax, 2)   \n\t"
764
765                         "movq 16(%0, %%eax, 4), %%mm1   \n\t" // YUYV YUYV(8)
766                         "movq 24(%0, %%eax, 4), %%mm2   \n\t" // YUYV YUYV(12)
767                         "movq %%mm1, %%mm3              \n\t" // YUYV YUYV(8)
768                         "movq %%mm2, %%mm4              \n\t" // YUYV YUYV(12)
769                         "psrlw $8, %%mm1                \n\t" // U0V0 U0V0(8)
770                         "psrlw $8, %%mm2                \n\t" // U0V0 U0V0(12)
771                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(8)
772                         "pand %%mm7, %%mm4              \n\t" // Y0Y0 Y0Y0(12)
773                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
774                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
775
776                         MOVNTQ" %%mm3, 8(%1, %%eax, 2)  \n\t"
777
778                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
779                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
780                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
781                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
782                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
783                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
784                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
785                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
786
787                         MOVNTQ" %%mm0, (%3, %%eax)      \n\t"
788                         MOVNTQ" %%mm2, (%2, %%eax)      \n\t"
789
790                         "addl $8, %%eax                 \n\t"
791                         "cmpl %4, %%eax                 \n\t"
792                         " jb 1b                         \n\t"
793                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "r" (chromWidth)
794                         : "memory", "%eax"
795                 );
796
797                 ydst += lumStride;
798                 src  += srcStride;
799
800                 asm volatile(
801                         "xorl %%eax, %%eax              \n\t"
802                         ".balign 16                     \n\t"
803                         "1:                             \n\t"
804                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
805                         "movq (%0, %%eax, 4), %%mm0     \n\t" // YUYV YUYV(0)
806                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // YUYV YUYV(4)
807                         "movq 16(%0, %%eax, 4), %%mm2   \n\t" // YUYV YUYV(8)
808                         "movq 24(%0, %%eax, 4), %%mm3   \n\t" // YUYV YUYV(12)
809                         "pand %%mm7, %%mm0              \n\t" // Y0Y0 Y0Y0(0)
810                         "pand %%mm7, %%mm1              \n\t" // Y0Y0 Y0Y0(4)
811                         "pand %%mm7, %%mm2              \n\t" // Y0Y0 Y0Y0(8)
812                         "pand %%mm7, %%mm3              \n\t" // Y0Y0 Y0Y0(12)
813                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
814                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
815
816                         MOVNTQ" %%mm0, (%1, %%eax, 2)   \n\t"
817                         MOVNTQ" %%mm2, 8(%1, %%eax, 2)  \n\t"
818
819                         "addl $8, %%eax                 \n\t"
820                         "cmpl %4, %%eax                 \n\t"
821                         " jb 1b                         \n\t"
822
823                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "r" (chromWidth)
824                         : "memory", "%eax"
825                 );
826 #else
827                 int i;
828                 for(i=0; i<chromWidth; i++)
829                 {
830                         ydst[2*i+0]     = src[4*i+0];
831                         udst[i]         = src[4*i+1];
832                         ydst[2*i+1]     = src[4*i+2];
833                         vdst[i]         = src[4*i+3];
834                 }
835                 ydst += lumStride;
836                 src  += srcStride;
837
838                 for(i=0; i<chromWidth; i++)
839                 {
840                         ydst[2*i+0]     = src[4*i+0];
841                         ydst[2*i+1]     = src[4*i+2];
842                 }
843 #endif
844                 udst += chromStride;
845                 vdst += chromStride;
846                 ydst += lumStride;
847                 src  += srcStride;
848         }
849 #ifdef HAVE_MMX
850 asm volatile(   EMMS" \n\t"
851                 SFENCE" \n\t"
852                 :::"memory");
853 #endif
854 }
855
856 /**
857  *
858  * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
859  * problem for anyone then tell me, and ill fix it)
860  * chrominance data is only taken from every secound line others are ignored FIXME write HQ version
861  */
862 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
863         unsigned int width, unsigned int height,
864         unsigned int lumStride, unsigned int chromStride, unsigned int srcStride)
865 {
866         int y;
867         const int chromWidth= width>>1;
868         for(y=0; y<height; y+=2)
869         {
870 #ifdef HAVE_MMX
871                 asm volatile(
872                         "xorl %%eax, %%eax              \n\t"
873                         "pcmpeqw %%mm7, %%mm7           \n\t"
874                         "psrlw $8, %%mm7                \n\t" // FF,00,FF,00...
875                         ".balign 16                     \n\t"
876                         "1:                             \n\t"
877                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
878                         "movq (%0, %%eax, 4), %%mm0     \n\t" // UYVY UYVY(0)
879                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // UYVY UYVY(4)
880                         "movq %%mm0, %%mm2              \n\t" // UYVY UYVY(0)
881                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(4)
882                         "pand %%mm7, %%mm0              \n\t" // U0V0 U0V0(0)
883                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(4)
884                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(0)
885                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(4)
886                         "packuswb %%mm1, %%mm0          \n\t" // UVUV UVUV(0)
887                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(0)
888
889                         MOVNTQ" %%mm2, (%1, %%eax, 2)   \n\t"
890
891                         "movq 16(%0, %%eax, 4), %%mm1   \n\t" // UYVY UYVY(8)
892                         "movq 24(%0, %%eax, 4), %%mm2   \n\t" // UYVY UYVY(12)
893                         "movq %%mm1, %%mm3              \n\t" // UYVY UYVY(8)
894                         "movq %%mm2, %%mm4              \n\t" // UYVY UYVY(12)
895                         "pand %%mm7, %%mm1              \n\t" // U0V0 U0V0(8)
896                         "pand %%mm7, %%mm2              \n\t" // U0V0 U0V0(12)
897                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(8)
898                         "psrlw $8, %%mm4                \n\t" // Y0Y0 Y0Y0(12)
899                         "packuswb %%mm2, %%mm1          \n\t" // UVUV UVUV(8)
900                         "packuswb %%mm4, %%mm3          \n\t" // YYYY YYYY(8)
901
902                         MOVNTQ" %%mm3, 8(%1, %%eax, 2)  \n\t"
903
904                         "movq %%mm0, %%mm2              \n\t" // UVUV UVUV(0)
905                         "movq %%mm1, %%mm3              \n\t" // UVUV UVUV(8)
906                         "psrlw $8, %%mm0                \n\t" // V0V0 V0V0(0)
907                         "psrlw $8, %%mm1                \n\t" // V0V0 V0V0(8)
908                         "pand %%mm7, %%mm2              \n\t" // U0U0 U0U0(0)
909                         "pand %%mm7, %%mm3              \n\t" // U0U0 U0U0(8)
910                         "packuswb %%mm1, %%mm0          \n\t" // VVVV VVVV(0)
911                         "packuswb %%mm3, %%mm2          \n\t" // UUUU UUUU(0)
912
913                         MOVNTQ" %%mm0, (%3, %%eax)      \n\t"
914                         MOVNTQ" %%mm2, (%2, %%eax)      \n\t"
915
916                         "addl $8, %%eax                 \n\t"
917                         "cmpl %4, %%eax                 \n\t"
918                         " jb 1b                         \n\t"
919                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "r" (chromWidth)
920                         : "memory", "%eax"
921                 );
922
923                 ydst += lumStride;
924                 src  += srcStride;
925
926                 asm volatile(
927                         "xorl %%eax, %%eax              \n\t"
928                         ".balign 16                     \n\t"
929                         "1:                             \n\t"
930                         PREFETCH" 64(%0, %%eax, 4)      \n\t"
931                         "movq (%0, %%eax, 4), %%mm0     \n\t" // YUYV YUYV(0)
932                         "movq 8(%0, %%eax, 4), %%mm1    \n\t" // YUYV YUYV(4)
933                         "movq 16(%0, %%eax, 4), %%mm2   \n\t" // YUYV YUYV(8)
934                         "movq 24(%0, %%eax, 4), %%mm3   \n\t" // YUYV YUYV(12)
935                         "psrlw $8, %%mm0                \n\t" // Y0Y0 Y0Y0(0)
936                         "psrlw $8, %%mm1                \n\t" // Y0Y0 Y0Y0(4)
937                         "psrlw $8, %%mm2                \n\t" // Y0Y0 Y0Y0(8)
938                         "psrlw $8, %%mm3                \n\t" // Y0Y0 Y0Y0(12)
939                         "packuswb %%mm1, %%mm0          \n\t" // YYYY YYYY(0)
940                         "packuswb %%mm3, %%mm2          \n\t" // YYYY YYYY(8)
941
942                         MOVNTQ" %%mm0, (%1, %%eax, 2)   \n\t"
943                         MOVNTQ" %%mm2, 8(%1, %%eax, 2)  \n\t"
944
945                         "addl $8, %%eax                 \n\t"
946                         "cmpl %4, %%eax                 \n\t"
947                         " jb 1b                         \n\t"
948
949                         ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "r" (chromWidth)
950                         : "memory", "%eax"
951                 );
952 #else
953                 int i;
954                 for(i=0; i<chromWidth; i++)
955                 {
956                         udst[i]         = src[4*i+0];
957                         ydst[2*i+0]     = src[4*i+1];
958                         vdst[i]         = src[4*i+2];
959                         ydst[2*i+1]     = src[4*i+3];
960                 }
961                 ydst += lumStride;
962                 src  += srcStride;
963
964                 for(i=0; i<chromWidth; i++)
965                 {
966                         ydst[2*i+0]     = src[4*i+1];
967                         ydst[2*i+1]     = src[4*i+3];
968                 }
969 #endif
970                 udst += chromStride;
971                 vdst += chromStride;
972                 ydst += lumStride;
973                 src  += srcStride;
974         }
975 #ifdef HAVE_MMX
976 asm volatile(   EMMS" \n\t"
977                 SFENCE" \n\t"
978                 :::"memory");
979 #endif
980 }
981
982 /**
983  *
984  * height should be a multiple of 2 and width should be a multiple of 2 (if this is a
985  * problem for anyone then tell me, and ill fix it)
986  * chrominance data is only taken from every secound line others are ignored in the C version FIXME write HQ version
987  */
988 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
989         unsigned int width, unsigned int height,
990         unsigned int lumStride, unsigned int chromStride, unsigned int srcStride)
991 {
992         int y;
993         const int chromWidth= width>>1;
994 #ifdef HAVE_MMX
995         for(y=0; y<height-2; y+=2)
996         {
997                 int i;
998                 for(i=0; i<2; i++)
999                 {
1000                         asm volatile(
1001                                 "movl %2, %%eax                 \n\t"
1002                                 "movq "MANGLE(bgr2YCoeff)", %%mm6               \n\t"
1003                                 "movq "MANGLE(w1111)", %%mm5            \n\t"
1004                                 "pxor %%mm7, %%mm7              \n\t"
1005                                 "leal (%%eax, %%eax, 2), %%ebx  \n\t"
1006                                 ".balign 16                     \n\t"
1007                                 "1:                             \n\t"
1008                                 PREFETCH" 64(%0, %%ebx)         \n\t"
1009                                 "movd (%0, %%ebx), %%mm0        \n\t"
1010                                 "movd 3(%0, %%ebx), %%mm1       \n\t"
1011                                 "punpcklbw %%mm7, %%mm0         \n\t"
1012                                 "punpcklbw %%mm7, %%mm1         \n\t"
1013                                 "movd 6(%0, %%ebx), %%mm2       \n\t"
1014                                 "movd 9(%0, %%ebx), %%mm3       \n\t"
1015                                 "punpcklbw %%mm7, %%mm2         \n\t"
1016                                 "punpcklbw %%mm7, %%mm3         \n\t"
1017                                 "pmaddwd %%mm6, %%mm0           \n\t"
1018                                 "pmaddwd %%mm6, %%mm1           \n\t"
1019                                 "pmaddwd %%mm6, %%mm2           \n\t"
1020                                 "pmaddwd %%mm6, %%mm3           \n\t"
1021 #ifndef FAST_BGR2YV12
1022                                 "psrad $8, %%mm0                \n\t"
1023                                 "psrad $8, %%mm1                \n\t"
1024                                 "psrad $8, %%mm2                \n\t"
1025                                 "psrad $8, %%mm3                \n\t"
1026 #endif
1027                                 "packssdw %%mm1, %%mm0          \n\t"
1028                                 "packssdw %%mm3, %%mm2          \n\t"
1029                                 "pmaddwd %%mm5, %%mm0           \n\t"
1030                                 "pmaddwd %%mm5, %%mm2           \n\t"
1031                                 "packssdw %%mm2, %%mm0          \n\t"
1032                                 "psraw $7, %%mm0                \n\t"
1033
1034                                 "movd 12(%0, %%ebx), %%mm4      \n\t"
1035                                 "movd 15(%0, %%ebx), %%mm1      \n\t"
1036                                 "punpcklbw %%mm7, %%mm4         \n\t"
1037                                 "punpcklbw %%mm7, %%mm1         \n\t"
1038                                 "movd 18(%0, %%ebx), %%mm2      \n\t"
1039                                 "movd 21(%0, %%ebx), %%mm3      \n\t"
1040                                 "punpcklbw %%mm7, %%mm2         \n\t"
1041                                 "punpcklbw %%mm7, %%mm3         \n\t"
1042                                 "pmaddwd %%mm6, %%mm4           \n\t"
1043                                 "pmaddwd %%mm6, %%mm1           \n\t"
1044                                 "pmaddwd %%mm6, %%mm2           \n\t"
1045                                 "pmaddwd %%mm6, %%mm3           \n\t"
1046 #ifndef FAST_BGR2YV12
1047                                 "psrad $8, %%mm4                \n\t"
1048                                 "psrad $8, %%mm1                \n\t"
1049                                 "psrad $8, %%mm2                \n\t"
1050                                 "psrad $8, %%mm3                \n\t"
1051 #endif
1052                                 "packssdw %%mm1, %%mm4          \n\t"
1053                                 "packssdw %%mm3, %%mm2          \n\t"
1054                                 "pmaddwd %%mm5, %%mm4           \n\t"
1055                                 "pmaddwd %%mm5, %%mm2           \n\t"
1056                                 "addl $24, %%ebx                \n\t"
1057                                 "packssdw %%mm2, %%mm4          \n\t"
1058                                 "psraw $7, %%mm4                \n\t"
1059
1060                                 "packuswb %%mm4, %%mm0          \n\t"
1061                                 "paddusb "MANGLE(bgr2YOffset)", %%mm0   \n\t"
1062
1063                                 MOVNTQ" %%mm0, (%1, %%eax)      \n\t"
1064                                 "addl $8, %%eax                 \n\t"
1065                                 " js 1b                         \n\t"
1066                                 : : "r" (src+width*3), "r" (ydst+width), "g" (-width)
1067                                 : "%eax", "%ebx"
1068                         );
1069                         ydst += lumStride;
1070                         src  += srcStride;
1071                 }
1072                 src -= srcStride*2;
1073                 asm volatile(
1074                         "movl %4, %%eax                 \n\t"
1075                         "movq "MANGLE(w1111)", %%mm5            \n\t"
1076                         "movq "MANGLE(bgr2UCoeff)", %%mm6               \n\t"
1077                         "pxor %%mm7, %%mm7              \n\t"
1078                         "leal (%%eax, %%eax, 2), %%ebx  \n\t"
1079                         "addl %%ebx, %%ebx              \n\t"
1080                         ".balign 16                     \n\t"
1081                         "1:                             \n\t"
1082                         PREFETCH" 64(%0, %%ebx)         \n\t"
1083                         PREFETCH" 64(%1, %%ebx)         \n\t"
1084 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
1085                         "movq (%0, %%ebx), %%mm0        \n\t"
1086                         "movq (%1, %%ebx), %%mm1        \n\t"
1087                         "movq 6(%0, %%ebx), %%mm2       \n\t"
1088                         "movq 6(%1, %%ebx), %%mm3       \n\t"
1089                         PAVGB" %%mm1, %%mm0             \n\t"
1090                         PAVGB" %%mm3, %%mm2             \n\t"
1091                         "movq %%mm0, %%mm1              \n\t"
1092                         "movq %%mm2, %%mm3              \n\t"
1093                         "psrlq $24, %%mm0               \n\t"
1094                         "psrlq $24, %%mm2               \n\t"
1095                         PAVGB" %%mm1, %%mm0             \n\t"
1096                         PAVGB" %%mm3, %%mm2             \n\t"
1097                         "punpcklbw %%mm7, %%mm0         \n\t"
1098                         "punpcklbw %%mm7, %%mm2         \n\t"
1099 #else
1100                         "movd (%0, %%ebx), %%mm0        \n\t"
1101                         "movd (%1, %%ebx), %%mm1        \n\t"
1102                         "movd 3(%0, %%ebx), %%mm2       \n\t"
1103                         "movd 3(%1, %%ebx), %%mm3       \n\t"
1104                         "punpcklbw %%mm7, %%mm0         \n\t"
1105                         "punpcklbw %%mm7, %%mm1         \n\t"
1106                         "punpcklbw %%mm7, %%mm2         \n\t"
1107                         "punpcklbw %%mm7, %%mm3         \n\t"
1108                         "paddw %%mm1, %%mm0             \n\t"
1109                         "paddw %%mm3, %%mm2             \n\t"
1110                         "paddw %%mm2, %%mm0             \n\t"
1111                         "movd 6(%0, %%ebx), %%mm4       \n\t"
1112                         "movd 6(%1, %%ebx), %%mm1       \n\t"
1113                         "movd 9(%0, %%ebx), %%mm2       \n\t"
1114                         "movd 9(%1, %%ebx), %%mm3       \n\t"
1115                         "punpcklbw %%mm7, %%mm4         \n\t"
1116                         "punpcklbw %%mm7, %%mm1         \n\t"
1117                         "punpcklbw %%mm7, %%mm2         \n\t"
1118                         "punpcklbw %%mm7, %%mm3         \n\t"
1119                         "paddw %%mm1, %%mm4             \n\t"
1120                         "paddw %%mm3, %%mm2             \n\t"
1121                         "paddw %%mm4, %%mm2             \n\t"
1122                         "psrlw $2, %%mm0                \n\t"
1123                         "psrlw $2, %%mm2                \n\t"
1124 #endif
1125                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
1126                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
1127
1128                         "pmaddwd %%mm0, %%mm1           \n\t"
1129                         "pmaddwd %%mm2, %%mm3           \n\t"
1130                         "pmaddwd %%mm6, %%mm0           \n\t"
1131                         "pmaddwd %%mm6, %%mm2           \n\t"
1132 #ifndef FAST_BGR2YV12
1133                         "psrad $8, %%mm0                \n\t"
1134                         "psrad $8, %%mm1                \n\t"
1135                         "psrad $8, %%mm2                \n\t"
1136                         "psrad $8, %%mm3                \n\t"
1137 #endif
1138                         "packssdw %%mm2, %%mm0          \n\t"
1139                         "packssdw %%mm3, %%mm1          \n\t"
1140                         "pmaddwd %%mm5, %%mm0           \n\t"
1141                         "pmaddwd %%mm5, %%mm1           \n\t"
1142                         "packssdw %%mm1, %%mm0          \n\t" // V1 V0 U1 U0
1143                         "psraw $7, %%mm0                \n\t"
1144
1145 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
1146                         "movq 12(%0, %%ebx), %%mm4      \n\t"
1147                         "movq 12(%1, %%ebx), %%mm1      \n\t"
1148                         "movq 18(%0, %%ebx), %%mm2      \n\t"
1149                         "movq 18(%1, %%ebx), %%mm3      \n\t"
1150                         PAVGB" %%mm1, %%mm4             \n\t"
1151                         PAVGB" %%mm3, %%mm2             \n\t"
1152                         "movq %%mm4, %%mm1              \n\t"
1153                         "movq %%mm2, %%mm3              \n\t"
1154                         "psrlq $24, %%mm4               \n\t"
1155                         "psrlq $24, %%mm2               \n\t"
1156                         PAVGB" %%mm1, %%mm4             \n\t"
1157                         PAVGB" %%mm3, %%mm2             \n\t"
1158                         "punpcklbw %%mm7, %%mm4         \n\t"
1159                         "punpcklbw %%mm7, %%mm2         \n\t"
1160 #else
1161                         "movd 12(%0, %%ebx), %%mm4      \n\t"
1162                         "movd 12(%1, %%ebx), %%mm1      \n\t"
1163                         "movd 15(%0, %%ebx), %%mm2      \n\t"
1164                         "movd 15(%1, %%ebx), %%mm3      \n\t"
1165                         "punpcklbw %%mm7, %%mm4         \n\t"
1166                         "punpcklbw %%mm7, %%mm1         \n\t"
1167                         "punpcklbw %%mm7, %%mm2         \n\t"
1168                         "punpcklbw %%mm7, %%mm3         \n\t"
1169                         "paddw %%mm1, %%mm4             \n\t"
1170                         "paddw %%mm3, %%mm2             \n\t"
1171                         "paddw %%mm2, %%mm4             \n\t"
1172                         "movd 18(%0, %%ebx), %%mm5      \n\t"
1173                         "movd 18(%1, %%ebx), %%mm1      \n\t"
1174                         "movd 21(%0, %%ebx), %%mm2      \n\t"
1175                         "movd 21(%1, %%ebx), %%mm3      \n\t"
1176                         "punpcklbw %%mm7, %%mm5         \n\t"
1177                         "punpcklbw %%mm7, %%mm1         \n\t"
1178                         "punpcklbw %%mm7, %%mm2         \n\t"
1179                         "punpcklbw %%mm7, %%mm3         \n\t"
1180                         "paddw %%mm1, %%mm5             \n\t"
1181                         "paddw %%mm3, %%mm2             \n\t"
1182                         "paddw %%mm5, %%mm2             \n\t"
1183                         "movq "MANGLE(w1111)", %%mm5            \n\t"
1184                         "psrlw $2, %%mm4                \n\t"
1185                         "psrlw $2, %%mm2                \n\t"
1186 #endif
1187                         "movq "MANGLE(bgr2VCoeff)", %%mm1               \n\t"
1188                         "movq "MANGLE(bgr2VCoeff)", %%mm3               \n\t"
1189
1190                         "pmaddwd %%mm4, %%mm1           \n\t"
1191                         "pmaddwd %%mm2, %%mm3           \n\t"
1192                         "pmaddwd %%mm6, %%mm4           \n\t"
1193                         "pmaddwd %%mm6, %%mm2           \n\t"
1194 #ifndef FAST_BGR2YV12
1195                         "psrad $8, %%mm4                \n\t"
1196                         "psrad $8, %%mm1                \n\t"
1197                         "psrad $8, %%mm2                \n\t"
1198                         "psrad $8, %%mm3                \n\t"
1199 #endif
1200                         "packssdw %%mm2, %%mm4          \n\t"
1201                         "packssdw %%mm3, %%mm1          \n\t"
1202                         "pmaddwd %%mm5, %%mm4           \n\t"
1203                         "pmaddwd %%mm5, %%mm1           \n\t"
1204                         "addl $24, %%ebx                \n\t"
1205                         "packssdw %%mm1, %%mm4          \n\t" // V3 V2 U3 U2
1206                         "psraw $7, %%mm4                \n\t"
1207
1208                         "movq %%mm0, %%mm1              \n\t"
1209                         "punpckldq %%mm4, %%mm0         \n\t"
1210                         "punpckhdq %%mm4, %%mm1         \n\t"
1211                         "packsswb %%mm1, %%mm0          \n\t"
1212                         "paddb "MANGLE(bgr2UVOffset)", %%mm0    \n\t"
1213
1214                         "movd %%mm0, (%2, %%eax)        \n\t"
1215                         "punpckhdq %%mm0, %%mm0         \n\t"
1216                         "movd %%mm0, (%3, %%eax)        \n\t"
1217                         "addl $4, %%eax                 \n\t"
1218                         " js 1b                         \n\t"
1219                         : : "r" (src+width*6), "r" (src+srcStride+width*6), "r" (udst+width), "r" (vdst+width), "g" (-width)
1220                         : "%eax", "%ebx"
1221                 );
1222
1223                 udst += chromStride;
1224                 vdst += chromStride;
1225                 src  += srcStride*2;
1226         }
1227
1228         asm volatile(   EMMS" \n\t"
1229                         SFENCE" \n\t"
1230                         :::"memory");
1231 #else
1232         y=0;
1233 #endif
1234         for(; y<height; y+=2)
1235         {
1236                 int i;
1237                 for(i=0; i<chromWidth; i++)
1238                 {
1239                         unsigned int b= src[6*i+0];
1240                         unsigned int g= src[6*i+1];
1241                         unsigned int r= src[6*i+2];
1242
1243                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
1244                         unsigned int V  =  ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128;
1245                         unsigned int U  =  ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128;
1246
1247                         udst[i]         = U;
1248                         vdst[i]         = V;
1249                         ydst[2*i]       = Y;
1250
1251                         b= src[6*i+3];
1252                         g= src[6*i+4];
1253                         r= src[6*i+5];
1254
1255                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
1256                         ydst[2*i+1]     = Y;
1257                 }
1258                 ydst += lumStride;
1259                 src  += srcStride;
1260
1261                 for(i=0; i<chromWidth; i++)
1262                 {
1263                         unsigned int b= src[6*i+0];
1264                         unsigned int g= src[6*i+1];
1265                         unsigned int r= src[6*i+2];
1266
1267                         unsigned int Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
1268
1269                         ydst[2*i]       = Y;
1270
1271                         b= src[6*i+3];
1272                         g= src[6*i+4];
1273                         r= src[6*i+5];
1274
1275                         Y  =  ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
1276                         ydst[2*i+1]     = Y;
1277                 }
1278                 udst += chromStride;
1279                 vdst += chromStride;
1280                 ydst += lumStride;
1281                 src  += srcStride;
1282         }
1283 }
1284
1285 void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest,
1286                             int width, int height, int src1Stride, int src2Stride, int dstStride){
1287         int h;
1288
1289         for(h=0; h < height; h++)
1290         {
1291                 int w;
1292
1293 #ifdef HAVE_MMX
1294 #ifdef HAVE_SSE2
1295                 asm(
1296                         "xorl %%eax, %%eax              \n\t"
1297                         "1:                             \n\t"
1298                         PREFETCH" 64(%1, %%eax)         \n\t"
1299                         PREFETCH" 64(%2, %%eax)         \n\t"
1300                         "movdqa (%1, %%eax), %%xmm0     \n\t"
1301                         "movdqa (%1, %%eax), %%xmm1     \n\t"
1302                         "movdqa (%2, %%eax), %%xmm2     \n\t"
1303                         "punpcklbw %%xmm2, %%xmm0       \n\t"
1304                         "punpckhbw %%xmm2, %%xmm1       \n\t"
1305                         "movntdq %%xmm0, (%0, %%eax, 2) \n\t"
1306                         "movntdq %%xmm1, 16(%0, %%eax, 2)\n\t"
1307                         "addl $16, %%eax                        \n\t"
1308                         "cmpl %3, %%eax                 \n\t"
1309                         " jb 1b                         \n\t"
1310                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
1311                         : "memory", "%eax"
1312                 );
1313 #else
1314                 asm(
1315                         "xorl %%eax, %%eax              \n\t"
1316                         "1:                             \n\t"
1317                         PREFETCH" 64(%1, %%eax)         \n\t"
1318                         PREFETCH" 64(%2, %%eax)         \n\t"
1319                         "movq (%1, %%eax), %%mm0        \n\t"
1320                         "movq 8(%1, %%eax), %%mm2       \n\t"
1321                         "movq %%mm0, %%mm1              \n\t"
1322                         "movq %%mm2, %%mm3              \n\t"
1323                         "movq (%2, %%eax), %%mm4        \n\t"
1324                         "movq 8(%2, %%eax), %%mm5       \n\t"
1325                         "punpcklbw %%mm4, %%mm0         \n\t"
1326                         "punpckhbw %%mm4, %%mm1         \n\t"
1327                         "punpcklbw %%mm5, %%mm2         \n\t"
1328                         "punpckhbw %%mm5, %%mm3         \n\t"
1329                         MOVNTQ" %%mm0, (%0, %%eax, 2)   \n\t"
1330                         MOVNTQ" %%mm1, 8(%0, %%eax, 2)  \n\t"
1331                         MOVNTQ" %%mm2, 16(%0, %%eax, 2) \n\t"
1332                         MOVNTQ" %%mm3, 24(%0, %%eax, 2) \n\t"
1333                         "addl $16, %%eax                        \n\t"
1334                         "cmpl %3, %%eax                 \n\t"
1335                         " jb 1b                         \n\t"
1336                         ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
1337                         : "memory", "%eax"
1338                 );
1339 #endif
1340                 for(w= (width&(~15)); w < width; w++)
1341                 {
1342                         dest[2*w+0] = src1[w];
1343                         dest[2*w+1] = src2[w];
1344                 }
1345 #else
1346                 for(w=0; w < width; w++)
1347                 {
1348                         dest[2*w+0] = src1[w];
1349                         dest[2*w+1] = src2[w];
1350                 }
1351 #endif
1352                 dest += dstStride;
1353                 src1 += src1Stride;
1354                 src2 += src2Stride;
1355         }
1356 #ifdef HAVE_MMX
1357         asm(
1358                 EMMS" \n\t"
1359                 SFENCE" \n\t"
1360                 ::: "memory"
1361                 );
1362 #endif
1363 }