3 * rgb2rgb.c, Software RGB to RGB convertor
4 * pluralize by Software PAL8 to RGB convertor
5 * Software YUV to YUV convertor
6 * Software YUV to RGB convertor
7 * Written by Nick Kurshev.
8 * palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
9 * lot of big-endian byteorder fixes by Alex Beregszaszi
11 * This file is part of FFmpeg.
13 * FFmpeg is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
18 * FFmpeg is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * You should have received a copy of the GNU General Public License
24 * along with FFmpeg; if not, write to the Free Software
25 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
27 * the C code (not assembly, mmx, ...) of this file can be used
28 * under the LGPL license too
32 #include <inttypes.h> /* for __WORDSIZE */
35 // #warning You have misconfigured system and probably will lose performance!
36 #define __WORDSIZE MP_WORDSIZE
54 #define PREFETCH "prefetch"
55 #define PREFETCHW "prefetchw"
56 #define PAVGB "pavgusb"
57 #elif defined ( HAVE_MMX2 )
58 #define PREFETCH "prefetchnta"
59 #define PREFETCHW "prefetcht0"
66 #define PREFETCH " # nop"
67 #define PREFETCHW " # nop"
72 /* On K6 femms is faster of emms. On K7 femms is directly mapped on emms. */
79 #define MOVNTQ "movntq"
80 #define SFENCE "sfence"
83 #define SFENCE " # nop"
86 static inline void RENAME(rgb24to32)(const uint8_t *src,uint8_t *dst,long src_size)
89 const uint8_t *s = src;
92 const uint8_t *mm_end;
96 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
98 __asm __volatile("movq %0, %%mm7"::"m"(mask32):"memory");
104 "punpckldq 3%1, %%mm0\n\t"
105 "movd 6%1, %%mm1\n\t"
106 "punpckldq 9%1, %%mm1\n\t"
107 "movd 12%1, %%mm2\n\t"
108 "punpckldq 15%1, %%mm2\n\t"
109 "movd 18%1, %%mm3\n\t"
110 "punpckldq 21%1, %%mm3\n\t"
111 "pand %%mm7, %%mm0\n\t"
112 "pand %%mm7, %%mm1\n\t"
113 "pand %%mm7, %%mm2\n\t"
114 "pand %%mm7, %%mm3\n\t"
115 MOVNTQ" %%mm0, %0\n\t"
116 MOVNTQ" %%mm1, 8%0\n\t"
117 MOVNTQ" %%mm2, 16%0\n\t"
125 __asm __volatile(SFENCE:::"memory");
126 __asm __volatile(EMMS:::"memory");
130 #ifdef WORDS_BIGENDIAN
131 /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */
146 static inline void RENAME(rgb32to24)(const uint8_t *src,uint8_t *dst,long src_size)
149 const uint8_t *s = src;
152 const uint8_t *mm_end;
156 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
163 "movq 8%1, %%mm1\n\t"
164 "movq 16%1, %%mm4\n\t"
165 "movq 24%1, %%mm5\n\t"
166 "movq %%mm0, %%mm2\n\t"
167 "movq %%mm1, %%mm3\n\t"
168 "movq %%mm4, %%mm6\n\t"
169 "movq %%mm5, %%mm7\n\t"
170 "psrlq $8, %%mm2\n\t"
171 "psrlq $8, %%mm3\n\t"
172 "psrlq $8, %%mm6\n\t"
173 "psrlq $8, %%mm7\n\t"
182 "por %%mm2, %%mm0\n\t"
183 "por %%mm3, %%mm1\n\t"
184 "por %%mm6, %%mm4\n\t"
185 "por %%mm7, %%mm5\n\t"
187 "movq %%mm1, %%mm2\n\t"
188 "movq %%mm4, %%mm3\n\t"
189 "psllq $48, %%mm2\n\t"
190 "psllq $32, %%mm3\n\t"
193 "por %%mm2, %%mm0\n\t"
194 "psrlq $16, %%mm1\n\t"
195 "psrlq $32, %%mm4\n\t"
196 "psllq $16, %%mm5\n\t"
197 "por %%mm3, %%mm1\n\t"
199 "por %%mm5, %%mm4\n\t"
201 MOVNTQ" %%mm0, %0\n\t"
202 MOVNTQ" %%mm1, 8%0\n\t"
205 :"m"(*s),"m"(mask24l),
206 "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
211 __asm __volatile(SFENCE:::"memory");
212 __asm __volatile(EMMS:::"memory");
216 #ifdef WORDS_BIGENDIAN
217 /* RGB32 (= A,B,G,R) -> RGB24 (= R,G,B) */
233 Original by Strepto/Astral
234 ported to gcc & bugfixed : A'rpi
235 MMX2, 3DNOW optimization by Nick Kurshev
236 32bit c version, and and&add trick by Michael Niedermayer
238 static inline void RENAME(rgb15to16)(const uint8_t *src,uint8_t *dst,long src_size)
240 register const uint8_t* s=src;
241 register uint8_t* d=dst;
242 register const uint8_t *end;
243 const uint8_t *mm_end;
246 __asm __volatile(PREFETCH" %0"::"m"(*s));
247 __asm __volatile("movq %0, %%mm4"::"m"(mask15s));
254 "movq 8%1, %%mm2\n\t"
255 "movq %%mm0, %%mm1\n\t"
256 "movq %%mm2, %%mm3\n\t"
257 "pand %%mm4, %%mm0\n\t"
258 "pand %%mm4, %%mm2\n\t"
259 "paddw %%mm1, %%mm0\n\t"
260 "paddw %%mm3, %%mm2\n\t"
261 MOVNTQ" %%mm0, %0\n\t"
269 __asm __volatile(SFENCE:::"memory");
270 __asm __volatile(EMMS:::"memory");
275 register unsigned x= *((uint32_t *)s);
276 *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
282 register unsigned short x= *((uint16_t *)s);
283 *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
287 static inline void RENAME(rgb16to15)(const uint8_t *src,uint8_t *dst,long src_size)
289 register const uint8_t* s=src;
290 register uint8_t* d=dst;
291 register const uint8_t *end;
292 const uint8_t *mm_end;
295 __asm __volatile(PREFETCH" %0"::"m"(*s));
296 __asm __volatile("movq %0, %%mm7"::"m"(mask15rg));
297 __asm __volatile("movq %0, %%mm6"::"m"(mask15b));
304 "movq 8%1, %%mm2\n\t"
305 "movq %%mm0, %%mm1\n\t"
306 "movq %%mm2, %%mm3\n\t"
307 "psrlq $1, %%mm0\n\t"
308 "psrlq $1, %%mm2\n\t"
309 "pand %%mm7, %%mm0\n\t"
310 "pand %%mm7, %%mm2\n\t"
311 "pand %%mm6, %%mm1\n\t"
312 "pand %%mm6, %%mm3\n\t"
313 "por %%mm1, %%mm0\n\t"
314 "por %%mm3, %%mm2\n\t"
315 MOVNTQ" %%mm0, %0\n\t"
323 __asm __volatile(SFENCE:::"memory");
324 __asm __volatile(EMMS:::"memory");
329 register uint32_t x= *((uint32_t *)s);
330 *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
336 register uint16_t x= *((uint16_t *)s);
337 *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
343 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, long src_size)
345 const uint8_t *s = src;
348 const uint8_t *mm_end;
350 uint16_t *d = (uint16_t *)dst;
354 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
356 "movq %3, %%mm5 \n\t"
357 "movq %4, %%mm6 \n\t"
358 "movq %5, %%mm7 \n\t"
361 PREFETCH" 32(%1) \n\t"
362 "movd (%1), %%mm0 \n\t"
363 "movd 4(%1), %%mm3 \n\t"
364 "punpckldq 8(%1), %%mm0 \n\t"
365 "punpckldq 12(%1), %%mm3 \n\t"
366 "movq %%mm0, %%mm1 \n\t"
367 "movq %%mm3, %%mm4 \n\t"
368 "pand %%mm6, %%mm0 \n\t"
369 "pand %%mm6, %%mm3 \n\t"
370 "pmaddwd %%mm7, %%mm0 \n\t"
371 "pmaddwd %%mm7, %%mm3 \n\t"
372 "pand %%mm5, %%mm1 \n\t"
373 "pand %%mm5, %%mm4 \n\t"
374 "por %%mm1, %%mm0 \n\t"
375 "por %%mm4, %%mm3 \n\t"
376 "psrld $5, %%mm0 \n\t"
377 "pslld $11, %%mm3 \n\t"
378 "por %%mm3, %%mm0 \n\t"
379 MOVNTQ" %%mm0, (%0) \n\t"
385 : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
388 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
392 ::"m"(red_16mask),"m"(green_16mask));
398 "movd 4%1, %%mm3\n\t"
399 "punpckldq 8%1, %%mm0\n\t"
400 "punpckldq 12%1, %%mm3\n\t"
401 "movq %%mm0, %%mm1\n\t"
402 "movq %%mm0, %%mm2\n\t"
403 "movq %%mm3, %%mm4\n\t"
404 "movq %%mm3, %%mm5\n\t"
405 "psrlq $3, %%mm0\n\t"
406 "psrlq $3, %%mm3\n\t"
409 "psrlq $5, %%mm1\n\t"
410 "psrlq $5, %%mm4\n\t"
411 "pand %%mm6, %%mm1\n\t"
412 "pand %%mm6, %%mm4\n\t"
413 "psrlq $8, %%mm2\n\t"
414 "psrlq $8, %%mm5\n\t"
415 "pand %%mm7, %%mm2\n\t"
416 "pand %%mm7, %%mm5\n\t"
417 "por %%mm1, %%mm0\n\t"
418 "por %%mm4, %%mm3\n\t"
419 "por %%mm2, %%mm0\n\t"
420 "por %%mm5, %%mm3\n\t"
421 "psllq $16, %%mm3\n\t"
422 "por %%mm3, %%mm0\n\t"
423 MOVNTQ" %%mm0, %0\n\t"
424 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
429 __asm __volatile(SFENCE:::"memory");
430 __asm __volatile(EMMS:::"memory");
434 register int rgb = *(uint32_t*)s; s += 4;
435 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
439 static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
441 const uint8_t *s = src;
444 const uint8_t *mm_end;
446 uint16_t *d = (uint16_t *)dst;
449 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
453 ::"m"(red_16mask),"m"(green_16mask));
460 "movd 4%1, %%mm3\n\t"
461 "punpckldq 8%1, %%mm0\n\t"
462 "punpckldq 12%1, %%mm3\n\t"
463 "movq %%mm0, %%mm1\n\t"
464 "movq %%mm0, %%mm2\n\t"
465 "movq %%mm3, %%mm4\n\t"
466 "movq %%mm3, %%mm5\n\t"
467 "psllq $8, %%mm0\n\t"
468 "psllq $8, %%mm3\n\t"
469 "pand %%mm7, %%mm0\n\t"
470 "pand %%mm7, %%mm3\n\t"
471 "psrlq $5, %%mm1\n\t"
472 "psrlq $5, %%mm4\n\t"
473 "pand %%mm6, %%mm1\n\t"
474 "pand %%mm6, %%mm4\n\t"
475 "psrlq $19, %%mm2\n\t"
476 "psrlq $19, %%mm5\n\t"
479 "por %%mm1, %%mm0\n\t"
480 "por %%mm4, %%mm3\n\t"
481 "por %%mm2, %%mm0\n\t"
482 "por %%mm5, %%mm3\n\t"
483 "psllq $16, %%mm3\n\t"
484 "por %%mm3, %%mm0\n\t"
485 MOVNTQ" %%mm0, %0\n\t"
486 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
490 __asm __volatile(SFENCE:::"memory");
491 __asm __volatile(EMMS:::"memory");
495 register int rgb = *(uint32_t*)s; s += 4;
496 *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
500 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, long src_size)
502 const uint8_t *s = src;
505 const uint8_t *mm_end;
507 uint16_t *d = (uint16_t *)dst;
511 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
513 "movq %3, %%mm5 \n\t"
514 "movq %4, %%mm6 \n\t"
515 "movq %5, %%mm7 \n\t"
518 PREFETCH" 32(%1) \n\t"
519 "movd (%1), %%mm0 \n\t"
520 "movd 4(%1), %%mm3 \n\t"
521 "punpckldq 8(%1), %%mm0 \n\t"
522 "punpckldq 12(%1), %%mm3 \n\t"
523 "movq %%mm0, %%mm1 \n\t"
524 "movq %%mm3, %%mm4 \n\t"
525 "pand %%mm6, %%mm0 \n\t"
526 "pand %%mm6, %%mm3 \n\t"
527 "pmaddwd %%mm7, %%mm0 \n\t"
528 "pmaddwd %%mm7, %%mm3 \n\t"
529 "pand %%mm5, %%mm1 \n\t"
530 "pand %%mm5, %%mm4 \n\t"
531 "por %%mm1, %%mm0 \n\t"
532 "por %%mm4, %%mm3 \n\t"
533 "psrld $6, %%mm0 \n\t"
534 "pslld $10, %%mm3 \n\t"
535 "por %%mm3, %%mm0 \n\t"
536 MOVNTQ" %%mm0, (%0) \n\t"
542 : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
545 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
549 ::"m"(red_15mask),"m"(green_15mask));
555 "movd 4%1, %%mm3\n\t"
556 "punpckldq 8%1, %%mm0\n\t"
557 "punpckldq 12%1, %%mm3\n\t"
558 "movq %%mm0, %%mm1\n\t"
559 "movq %%mm0, %%mm2\n\t"
560 "movq %%mm3, %%mm4\n\t"
561 "movq %%mm3, %%mm5\n\t"
562 "psrlq $3, %%mm0\n\t"
563 "psrlq $3, %%mm3\n\t"
566 "psrlq $6, %%mm1\n\t"
567 "psrlq $6, %%mm4\n\t"
568 "pand %%mm6, %%mm1\n\t"
569 "pand %%mm6, %%mm4\n\t"
570 "psrlq $9, %%mm2\n\t"
571 "psrlq $9, %%mm5\n\t"
572 "pand %%mm7, %%mm2\n\t"
573 "pand %%mm7, %%mm5\n\t"
574 "por %%mm1, %%mm0\n\t"
575 "por %%mm4, %%mm3\n\t"
576 "por %%mm2, %%mm0\n\t"
577 "por %%mm5, %%mm3\n\t"
578 "psllq $16, %%mm3\n\t"
579 "por %%mm3, %%mm0\n\t"
580 MOVNTQ" %%mm0, %0\n\t"
581 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
586 __asm __volatile(SFENCE:::"memory");
587 __asm __volatile(EMMS:::"memory");
591 register int rgb = *(uint32_t*)s; s += 4;
592 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
596 static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
598 const uint8_t *s = src;
601 const uint8_t *mm_end;
603 uint16_t *d = (uint16_t *)dst;
606 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
610 ::"m"(red_15mask),"m"(green_15mask));
617 "movd 4%1, %%mm3\n\t"
618 "punpckldq 8%1, %%mm0\n\t"
619 "punpckldq 12%1, %%mm3\n\t"
620 "movq %%mm0, %%mm1\n\t"
621 "movq %%mm0, %%mm2\n\t"
622 "movq %%mm3, %%mm4\n\t"
623 "movq %%mm3, %%mm5\n\t"
624 "psllq $7, %%mm0\n\t"
625 "psllq $7, %%mm3\n\t"
626 "pand %%mm7, %%mm0\n\t"
627 "pand %%mm7, %%mm3\n\t"
628 "psrlq $6, %%mm1\n\t"
629 "psrlq $6, %%mm4\n\t"
630 "pand %%mm6, %%mm1\n\t"
631 "pand %%mm6, %%mm4\n\t"
632 "psrlq $19, %%mm2\n\t"
633 "psrlq $19, %%mm5\n\t"
636 "por %%mm1, %%mm0\n\t"
637 "por %%mm4, %%mm3\n\t"
638 "por %%mm2, %%mm0\n\t"
639 "por %%mm5, %%mm3\n\t"
640 "psllq $16, %%mm3\n\t"
641 "por %%mm3, %%mm0\n\t"
642 MOVNTQ" %%mm0, %0\n\t"
643 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
647 __asm __volatile(SFENCE:::"memory");
648 __asm __volatile(EMMS:::"memory");
652 register int rgb = *(uint32_t*)s; s += 4;
653 *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
657 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, long src_size)
659 const uint8_t *s = src;
662 const uint8_t *mm_end;
664 uint16_t *d = (uint16_t *)dst;
667 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
671 ::"m"(red_16mask),"m"(green_16mask));
678 "movd 3%1, %%mm3\n\t"
679 "punpckldq 6%1, %%mm0\n\t"
680 "punpckldq 9%1, %%mm3\n\t"
681 "movq %%mm0, %%mm1\n\t"
682 "movq %%mm0, %%mm2\n\t"
683 "movq %%mm3, %%mm4\n\t"
684 "movq %%mm3, %%mm5\n\t"
685 "psrlq $3, %%mm0\n\t"
686 "psrlq $3, %%mm3\n\t"
689 "psrlq $5, %%mm1\n\t"
690 "psrlq $5, %%mm4\n\t"
691 "pand %%mm6, %%mm1\n\t"
692 "pand %%mm6, %%mm4\n\t"
693 "psrlq $8, %%mm2\n\t"
694 "psrlq $8, %%mm5\n\t"
695 "pand %%mm7, %%mm2\n\t"
696 "pand %%mm7, %%mm5\n\t"
697 "por %%mm1, %%mm0\n\t"
698 "por %%mm4, %%mm3\n\t"
699 "por %%mm2, %%mm0\n\t"
700 "por %%mm5, %%mm3\n\t"
701 "psllq $16, %%mm3\n\t"
702 "por %%mm3, %%mm0\n\t"
703 MOVNTQ" %%mm0, %0\n\t"
704 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
708 __asm __volatile(SFENCE:::"memory");
709 __asm __volatile(EMMS:::"memory");
716 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
720 static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
722 const uint8_t *s = src;
725 const uint8_t *mm_end;
727 uint16_t *d = (uint16_t *)dst;
730 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
734 ::"m"(red_16mask),"m"(green_16mask));
741 "movd 3%1, %%mm3\n\t"
742 "punpckldq 6%1, %%mm0\n\t"
743 "punpckldq 9%1, %%mm3\n\t"
744 "movq %%mm0, %%mm1\n\t"
745 "movq %%mm0, %%mm2\n\t"
746 "movq %%mm3, %%mm4\n\t"
747 "movq %%mm3, %%mm5\n\t"
748 "psllq $8, %%mm0\n\t"
749 "psllq $8, %%mm3\n\t"
750 "pand %%mm7, %%mm0\n\t"
751 "pand %%mm7, %%mm3\n\t"
752 "psrlq $5, %%mm1\n\t"
753 "psrlq $5, %%mm4\n\t"
754 "pand %%mm6, %%mm1\n\t"
755 "pand %%mm6, %%mm4\n\t"
756 "psrlq $19, %%mm2\n\t"
757 "psrlq $19, %%mm5\n\t"
760 "por %%mm1, %%mm0\n\t"
761 "por %%mm4, %%mm3\n\t"
762 "por %%mm2, %%mm0\n\t"
763 "por %%mm5, %%mm3\n\t"
764 "psllq $16, %%mm3\n\t"
765 "por %%mm3, %%mm0\n\t"
766 MOVNTQ" %%mm0, %0\n\t"
767 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
771 __asm __volatile(SFENCE:::"memory");
772 __asm __volatile(EMMS:::"memory");
779 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
783 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, long src_size)
785 const uint8_t *s = src;
788 const uint8_t *mm_end;
790 uint16_t *d = (uint16_t *)dst;
793 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
797 ::"m"(red_15mask),"m"(green_15mask));
804 "movd 3%1, %%mm3\n\t"
805 "punpckldq 6%1, %%mm0\n\t"
806 "punpckldq 9%1, %%mm3\n\t"
807 "movq %%mm0, %%mm1\n\t"
808 "movq %%mm0, %%mm2\n\t"
809 "movq %%mm3, %%mm4\n\t"
810 "movq %%mm3, %%mm5\n\t"
811 "psrlq $3, %%mm0\n\t"
812 "psrlq $3, %%mm3\n\t"
815 "psrlq $6, %%mm1\n\t"
816 "psrlq $6, %%mm4\n\t"
817 "pand %%mm6, %%mm1\n\t"
818 "pand %%mm6, %%mm4\n\t"
819 "psrlq $9, %%mm2\n\t"
820 "psrlq $9, %%mm5\n\t"
821 "pand %%mm7, %%mm2\n\t"
822 "pand %%mm7, %%mm5\n\t"
823 "por %%mm1, %%mm0\n\t"
824 "por %%mm4, %%mm3\n\t"
825 "por %%mm2, %%mm0\n\t"
826 "por %%mm5, %%mm3\n\t"
827 "psllq $16, %%mm3\n\t"
828 "por %%mm3, %%mm0\n\t"
829 MOVNTQ" %%mm0, %0\n\t"
830 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
834 __asm __volatile(SFENCE:::"memory");
835 __asm __volatile(EMMS:::"memory");
842 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
846 static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
848 const uint8_t *s = src;
851 const uint8_t *mm_end;
853 uint16_t *d = (uint16_t *)dst;
856 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
860 ::"m"(red_15mask),"m"(green_15mask));
867 "movd 3%1, %%mm3\n\t"
868 "punpckldq 6%1, %%mm0\n\t"
869 "punpckldq 9%1, %%mm3\n\t"
870 "movq %%mm0, %%mm1\n\t"
871 "movq %%mm0, %%mm2\n\t"
872 "movq %%mm3, %%mm4\n\t"
873 "movq %%mm3, %%mm5\n\t"
874 "psllq $7, %%mm0\n\t"
875 "psllq $7, %%mm3\n\t"
876 "pand %%mm7, %%mm0\n\t"
877 "pand %%mm7, %%mm3\n\t"
878 "psrlq $6, %%mm1\n\t"
879 "psrlq $6, %%mm4\n\t"
880 "pand %%mm6, %%mm1\n\t"
881 "pand %%mm6, %%mm4\n\t"
882 "psrlq $19, %%mm2\n\t"
883 "psrlq $19, %%mm5\n\t"
886 "por %%mm1, %%mm0\n\t"
887 "por %%mm4, %%mm3\n\t"
888 "por %%mm2, %%mm0\n\t"
889 "por %%mm5, %%mm3\n\t"
890 "psllq $16, %%mm3\n\t"
891 "por %%mm3, %%mm0\n\t"
892 MOVNTQ" %%mm0, %0\n\t"
893 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
897 __asm __volatile(SFENCE:::"memory");
898 __asm __volatile(EMMS:::"memory");
905 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
910 I use here less accurate approximation by simply
911 left-shifting the input
912 value and filling the low order bits with
913 zeroes. This method improves png's
914 compression but this scheme cannot reproduce white exactly, since it does not
915 generate an all-ones maximum value; the net effect is to darken the
918 The better method should be "left bit replication":
928 | Leftmost Bits Repeated to Fill Open Bits
932 static inline void RENAME(rgb15to24)(const uint8_t *src, uint8_t *dst, long src_size)
936 const uint16_t *mm_end;
938 uint8_t *d = (uint8_t *)dst;
939 const uint16_t *s = (uint16_t *)src;
940 end = s + src_size/2;
942 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
954 "psllq $3, %%mm0\n\t"
955 "psrlq $2, %%mm1\n\t"
956 "psrlq $7, %%mm2\n\t"
957 "movq %%mm0, %%mm3\n\t"
958 "movq %%mm1, %%mm4\n\t"
959 "movq %%mm2, %%mm5\n\t"
960 "punpcklwd %5, %%mm0\n\t"
961 "punpcklwd %5, %%mm1\n\t"
962 "punpcklwd %5, %%mm2\n\t"
963 "punpckhwd %5, %%mm3\n\t"
964 "punpckhwd %5, %%mm4\n\t"
965 "punpckhwd %5, %%mm5\n\t"
966 "psllq $8, %%mm1\n\t"
967 "psllq $16, %%mm2\n\t"
968 "por %%mm1, %%mm0\n\t"
969 "por %%mm2, %%mm0\n\t"
970 "psllq $8, %%mm4\n\t"
971 "psllq $16, %%mm5\n\t"
972 "por %%mm4, %%mm3\n\t"
973 "por %%mm5, %%mm3\n\t"
975 "movq %%mm0, %%mm6\n\t"
976 "movq %%mm3, %%mm7\n\t"
978 "movq 8%1, %%mm0\n\t"
979 "movq 8%1, %%mm1\n\t"
980 "movq 8%1, %%mm2\n\t"
984 "psllq $3, %%mm0\n\t"
985 "psrlq $2, %%mm1\n\t"
986 "psrlq $7, %%mm2\n\t"
987 "movq %%mm0, %%mm3\n\t"
988 "movq %%mm1, %%mm4\n\t"
989 "movq %%mm2, %%mm5\n\t"
990 "punpcklwd %5, %%mm0\n\t"
991 "punpcklwd %5, %%mm1\n\t"
992 "punpcklwd %5, %%mm2\n\t"
993 "punpckhwd %5, %%mm3\n\t"
994 "punpckhwd %5, %%mm4\n\t"
995 "punpckhwd %5, %%mm5\n\t"
996 "psllq $8, %%mm1\n\t"
997 "psllq $16, %%mm2\n\t"
998 "por %%mm1, %%mm0\n\t"
999 "por %%mm2, %%mm0\n\t"
1000 "psllq $8, %%mm4\n\t"
1001 "psllq $16, %%mm5\n\t"
1002 "por %%mm4, %%mm3\n\t"
1003 "por %%mm5, %%mm3\n\t"
1006 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)
1008 /* Borrowed 32 to 24 */
1010 "movq %%mm0, %%mm4\n\t"
1011 "movq %%mm3, %%mm5\n\t"
1012 "movq %%mm6, %%mm0\n\t"
1013 "movq %%mm7, %%mm1\n\t"
1015 "movq %%mm4, %%mm6\n\t"
1016 "movq %%mm5, %%mm7\n\t"
1017 "movq %%mm0, %%mm2\n\t"
1018 "movq %%mm1, %%mm3\n\t"
1020 "psrlq $8, %%mm2\n\t"
1021 "psrlq $8, %%mm3\n\t"
1022 "psrlq $8, %%mm6\n\t"
1023 "psrlq $8, %%mm7\n\t"
1024 "pand %2, %%mm0\n\t"
1025 "pand %2, %%mm1\n\t"
1026 "pand %2, %%mm4\n\t"
1027 "pand %2, %%mm5\n\t"
1028 "pand %3, %%mm2\n\t"
1029 "pand %3, %%mm3\n\t"
1030 "pand %3, %%mm6\n\t"
1031 "pand %3, %%mm7\n\t"
1032 "por %%mm2, %%mm0\n\t"
1033 "por %%mm3, %%mm1\n\t"
1034 "por %%mm6, %%mm4\n\t"
1035 "por %%mm7, %%mm5\n\t"
1037 "movq %%mm1, %%mm2\n\t"
1038 "movq %%mm4, %%mm3\n\t"
1039 "psllq $48, %%mm2\n\t"
1040 "psllq $32, %%mm3\n\t"
1041 "pand %4, %%mm2\n\t"
1042 "pand %5, %%mm3\n\t"
1043 "por %%mm2, %%mm0\n\t"
1044 "psrlq $16, %%mm1\n\t"
1045 "psrlq $32, %%mm4\n\t"
1046 "psllq $16, %%mm5\n\t"
1047 "por %%mm3, %%mm1\n\t"
1048 "pand %6, %%mm5\n\t"
1049 "por %%mm5, %%mm4\n\t"
1051 MOVNTQ" %%mm0, %0\n\t"
1052 MOVNTQ" %%mm1, 8%0\n\t"
1053 MOVNTQ" %%mm4, 16%0"
1056 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1061 __asm __volatile(SFENCE:::"memory");
1062 __asm __volatile(EMMS:::"memory");
1066 register uint16_t bgr;
1068 *d++ = (bgr&0x1F)<<3;
1069 *d++ = (bgr&0x3E0)>>2;
1070 *d++ = (bgr&0x7C00)>>7;
1074 static inline void RENAME(rgb16to24)(const uint8_t *src, uint8_t *dst, long src_size)
1076 const uint16_t *end;
1078 const uint16_t *mm_end;
1080 uint8_t *d = (uint8_t *)dst;
1081 const uint16_t *s = (const uint16_t *)src;
1082 end = s + src_size/2;
1084 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
1090 "movq %1, %%mm0\n\t"
1091 "movq %1, %%mm1\n\t"
1092 "movq %1, %%mm2\n\t"
1093 "pand %2, %%mm0\n\t"
1094 "pand %3, %%mm1\n\t"
1095 "pand %4, %%mm2\n\t"
1096 "psllq $3, %%mm0\n\t"
1097 "psrlq $3, %%mm1\n\t"
1098 "psrlq $8, %%mm2\n\t"
1099 "movq %%mm0, %%mm3\n\t"
1100 "movq %%mm1, %%mm4\n\t"
1101 "movq %%mm2, %%mm5\n\t"
1102 "punpcklwd %5, %%mm0\n\t"
1103 "punpcklwd %5, %%mm1\n\t"
1104 "punpcklwd %5, %%mm2\n\t"
1105 "punpckhwd %5, %%mm3\n\t"
1106 "punpckhwd %5, %%mm4\n\t"
1107 "punpckhwd %5, %%mm5\n\t"
1108 "psllq $8, %%mm1\n\t"
1109 "psllq $16, %%mm2\n\t"
1110 "por %%mm1, %%mm0\n\t"
1111 "por %%mm2, %%mm0\n\t"
1112 "psllq $8, %%mm4\n\t"
1113 "psllq $16, %%mm5\n\t"
1114 "por %%mm4, %%mm3\n\t"
1115 "por %%mm5, %%mm3\n\t"
1117 "movq %%mm0, %%mm6\n\t"
1118 "movq %%mm3, %%mm7\n\t"
1120 "movq 8%1, %%mm0\n\t"
1121 "movq 8%1, %%mm1\n\t"
1122 "movq 8%1, %%mm2\n\t"
1123 "pand %2, %%mm0\n\t"
1124 "pand %3, %%mm1\n\t"
1125 "pand %4, %%mm2\n\t"
1126 "psllq $3, %%mm0\n\t"
1127 "psrlq $3, %%mm1\n\t"
1128 "psrlq $8, %%mm2\n\t"
1129 "movq %%mm0, %%mm3\n\t"
1130 "movq %%mm1, %%mm4\n\t"
1131 "movq %%mm2, %%mm5\n\t"
1132 "punpcklwd %5, %%mm0\n\t"
1133 "punpcklwd %5, %%mm1\n\t"
1134 "punpcklwd %5, %%mm2\n\t"
1135 "punpckhwd %5, %%mm3\n\t"
1136 "punpckhwd %5, %%mm4\n\t"
1137 "punpckhwd %5, %%mm5\n\t"
1138 "psllq $8, %%mm1\n\t"
1139 "psllq $16, %%mm2\n\t"
1140 "por %%mm1, %%mm0\n\t"
1141 "por %%mm2, %%mm0\n\t"
1142 "psllq $8, %%mm4\n\t"
1143 "psllq $16, %%mm5\n\t"
1144 "por %%mm4, %%mm3\n\t"
1145 "por %%mm5, %%mm3\n\t"
1147 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)
1149 /* Borrowed 32 to 24 */
1151 "movq %%mm0, %%mm4\n\t"
1152 "movq %%mm3, %%mm5\n\t"
1153 "movq %%mm6, %%mm0\n\t"
1154 "movq %%mm7, %%mm1\n\t"
1156 "movq %%mm4, %%mm6\n\t"
1157 "movq %%mm5, %%mm7\n\t"
1158 "movq %%mm0, %%mm2\n\t"
1159 "movq %%mm1, %%mm3\n\t"
1161 "psrlq $8, %%mm2\n\t"
1162 "psrlq $8, %%mm3\n\t"
1163 "psrlq $8, %%mm6\n\t"
1164 "psrlq $8, %%mm7\n\t"
1165 "pand %2, %%mm0\n\t"
1166 "pand %2, %%mm1\n\t"
1167 "pand %2, %%mm4\n\t"
1168 "pand %2, %%mm5\n\t"
1169 "pand %3, %%mm2\n\t"
1170 "pand %3, %%mm3\n\t"
1171 "pand %3, %%mm6\n\t"
1172 "pand %3, %%mm7\n\t"
1173 "por %%mm2, %%mm0\n\t"
1174 "por %%mm3, %%mm1\n\t"
1175 "por %%mm6, %%mm4\n\t"
1176 "por %%mm7, %%mm5\n\t"
1178 "movq %%mm1, %%mm2\n\t"
1179 "movq %%mm4, %%mm3\n\t"
1180 "psllq $48, %%mm2\n\t"
1181 "psllq $32, %%mm3\n\t"
1182 "pand %4, %%mm2\n\t"
1183 "pand %5, %%mm3\n\t"
1184 "por %%mm2, %%mm0\n\t"
1185 "psrlq $16, %%mm1\n\t"
1186 "psrlq $32, %%mm4\n\t"
1187 "psllq $16, %%mm5\n\t"
1188 "por %%mm3, %%mm1\n\t"
1189 "pand %6, %%mm5\n\t"
1190 "por %%mm5, %%mm4\n\t"
1192 MOVNTQ" %%mm0, %0\n\t"
1193 MOVNTQ" %%mm1, 8%0\n\t"
1194 MOVNTQ" %%mm4, 16%0"
1197 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1202 __asm __volatile(SFENCE:::"memory");
1203 __asm __volatile(EMMS:::"memory");
1207 register uint16_t bgr;
1209 *d++ = (bgr&0x1F)<<3;
1210 *d++ = (bgr&0x7E0)>>3;
1211 *d++ = (bgr&0xF800)>>8;
1215 static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_size)
1217 const uint16_t *end;
1219 const uint16_t *mm_end;
1221 uint8_t *d = (uint8_t *)dst;
1222 const uint16_t *s = (const uint16_t *)src;
1223 end = s + src_size/2;
1225 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
1226 __asm __volatile("pxor %%mm7,%%mm7\n\t":::"memory");
1232 "movq %1, %%mm0\n\t"
1233 "movq %1, %%mm1\n\t"
1234 "movq %1, %%mm2\n\t"
1235 "pand %2, %%mm0\n\t"
1236 "pand %3, %%mm1\n\t"
1237 "pand %4, %%mm2\n\t"
1238 "psllq $3, %%mm0\n\t"
1239 "psrlq $2, %%mm1\n\t"
1240 "psrlq $7, %%mm2\n\t"
1241 "movq %%mm0, %%mm3\n\t"
1242 "movq %%mm1, %%mm4\n\t"
1243 "movq %%mm2, %%mm5\n\t"
1244 "punpcklwd %%mm7, %%mm0\n\t"
1245 "punpcklwd %%mm7, %%mm1\n\t"
1246 "punpcklwd %%mm7, %%mm2\n\t"
1247 "punpckhwd %%mm7, %%mm3\n\t"
1248 "punpckhwd %%mm7, %%mm4\n\t"
1249 "punpckhwd %%mm7, %%mm5\n\t"
1250 "psllq $8, %%mm1\n\t"
1251 "psllq $16, %%mm2\n\t"
1252 "por %%mm1, %%mm0\n\t"
1253 "por %%mm2, %%mm0\n\t"
1254 "psllq $8, %%mm4\n\t"
1255 "psllq $16, %%mm5\n\t"
1256 "por %%mm4, %%mm3\n\t"
1257 "por %%mm5, %%mm3\n\t"
1258 MOVNTQ" %%mm0, %0\n\t"
1259 MOVNTQ" %%mm3, 8%0\n\t"
1261 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r)
1266 __asm __volatile(SFENCE:::"memory");
1267 __asm __volatile(EMMS:::"memory");
1271 #if 0 //slightly slower on athlon
1273 *((uint32_t*)d)++ = ((bgr&0x1F)<<3) + ((bgr&0x3E0)<<6) + ((bgr&0x7C00)<<9);
1275 register uint16_t bgr;
1277 #ifdef WORDS_BIGENDIAN
1279 *d++ = (bgr&0x7C00)>>7;
1280 *d++ = (bgr&0x3E0)>>2;
1281 *d++ = (bgr&0x1F)<<3;
1283 *d++ = (bgr&0x1F)<<3;
1284 *d++ = (bgr&0x3E0)>>2;
1285 *d++ = (bgr&0x7C00)>>7;
1293 static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_size)
1295 const uint16_t *end;
1297 const uint16_t *mm_end;
1299 uint8_t *d = (uint8_t *)dst;
1300 const uint16_t *s = (uint16_t *)src;
1301 end = s + src_size/2;
1303 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
1304 __asm __volatile("pxor %%mm7,%%mm7\n\t":::"memory");
1310 "movq %1, %%mm0\n\t"
1311 "movq %1, %%mm1\n\t"
1312 "movq %1, %%mm2\n\t"
1313 "pand %2, %%mm0\n\t"
1314 "pand %3, %%mm1\n\t"
1315 "pand %4, %%mm2\n\t"
1316 "psllq $3, %%mm0\n\t"
1317 "psrlq $3, %%mm1\n\t"
1318 "psrlq $8, %%mm2\n\t"
1319 "movq %%mm0, %%mm3\n\t"
1320 "movq %%mm1, %%mm4\n\t"
1321 "movq %%mm2, %%mm5\n\t"
1322 "punpcklwd %%mm7, %%mm0\n\t"
1323 "punpcklwd %%mm7, %%mm1\n\t"
1324 "punpcklwd %%mm7, %%mm2\n\t"
1325 "punpckhwd %%mm7, %%mm3\n\t"
1326 "punpckhwd %%mm7, %%mm4\n\t"
1327 "punpckhwd %%mm7, %%mm5\n\t"
1328 "psllq $8, %%mm1\n\t"
1329 "psllq $16, %%mm2\n\t"
1330 "por %%mm1, %%mm0\n\t"
1331 "por %%mm2, %%mm0\n\t"
1332 "psllq $8, %%mm4\n\t"
1333 "psllq $16, %%mm5\n\t"
1334 "por %%mm4, %%mm3\n\t"
1335 "por %%mm5, %%mm3\n\t"
1336 MOVNTQ" %%mm0, %0\n\t"
1337 MOVNTQ" %%mm3, 8%0\n\t"
1339 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r)
1344 __asm __volatile(SFENCE:::"memory");
1345 __asm __volatile(EMMS:::"memory");
1349 register uint16_t bgr;
1351 #ifdef WORDS_BIGENDIAN
1353 *d++ = (bgr&0xF800)>>8;
1354 *d++ = (bgr&0x7E0)>>3;
1355 *d++ = (bgr&0x1F)<<3;
1357 *d++ = (bgr&0x1F)<<3;
1358 *d++ = (bgr&0x7E0)>>3;
1359 *d++ = (bgr&0xF800)>>8;
1365 static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, long src_size)
1367 uint8_t *d = dst, *s = (uint8_t *) src;
1368 const uint8_t *end = s + src_size;
1371 " "PREFETCH" (%1) \n"
1372 " movq %3, %%mm7 \n"
1373 " pxor %4, %%mm7 \n"
1374 " movq %%mm7, %%mm6 \n"
1375 " pxor %5, %%mm7 \n"
1379 " "PREFETCH" 32(%1) \n"
1380 " movq (%1), %%mm0 \n"
1381 " movq 8(%1), %%mm1 \n"
1383 " pshufw $177, %%mm0, %%mm3 \n"
1384 " pshufw $177, %%mm1, %%mm5 \n"
1385 " pand %%mm7, %%mm0 \n"
1386 " pand %%mm6, %%mm3 \n"
1387 " pand %%mm7, %%mm1 \n"
1388 " pand %%mm6, %%mm5 \n"
1389 " por %%mm3, %%mm0 \n"
1390 " por %%mm5, %%mm1 \n"
1392 " movq %%mm0, %%mm2 \n"
1393 " movq %%mm1, %%mm4 \n"
1394 " pand %%mm7, %%mm0 \n"
1395 " pand %%mm6, %%mm2 \n"
1396 " pand %%mm7, %%mm1 \n"
1397 " pand %%mm6, %%mm4 \n"
1398 " movq %%mm2, %%mm3 \n"
1399 " movq %%mm4, %%mm5 \n"
1400 " pslld $16, %%mm2 \n"
1401 " psrld $16, %%mm3 \n"
1402 " pslld $16, %%mm4 \n"
1403 " psrld $16, %%mm5 \n"
1404 " por %%mm2, %%mm0 \n"
1405 " por %%mm4, %%mm1 \n"
1406 " por %%mm3, %%mm0 \n"
1407 " por %%mm5, %%mm1 \n"
1409 " "MOVNTQ" %%mm0, (%0) \n"
1410 " "MOVNTQ" %%mm1, 8(%0) \n"
1419 : "r" (end-15), "m" (mask32b), "m" (mask32r), "m" (mmx_one)
1422 for (; s<end; s+=4, d+=4) {
1423 int v = *(uint32_t *)s, g = v & 0xff00;
1425 *(uint32_t *)d = (v>>16) + g + (v<<16);
1429 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, long src_size)
1433 long mmx_size= 23 - src_size;
1435 "movq "MANGLE(mask24r)", %%mm5 \n\t"
1436 "movq "MANGLE(mask24g)", %%mm6 \n\t"
1437 "movq "MANGLE(mask24b)", %%mm7 \n\t"
1440 PREFETCH" 32(%1, %%"REG_a") \n\t"
1441 "movq (%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1442 "movq (%1, %%"REG_a"), %%mm1 \n\t" // BGR BGR BG
1443 "movq 2(%1, %%"REG_a"), %%mm2 \n\t" // R BGR BGR B
1444 "psllq $16, %%mm0 \n\t" // 00 BGR BGR
1445 "pand %%mm5, %%mm0 \n\t"
1446 "pand %%mm6, %%mm1 \n\t"
1447 "pand %%mm7, %%mm2 \n\t"
1448 "por %%mm0, %%mm1 \n\t"
1449 "por %%mm2, %%mm1 \n\t"
1450 "movq 6(%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1451 MOVNTQ" %%mm1, (%2, %%"REG_a")\n\t" // RGB RGB RG
1452 "movq 8(%1, %%"REG_a"), %%mm1 \n\t" // R BGR BGR B
1453 "movq 10(%1, %%"REG_a"), %%mm2 \n\t" // GR BGR BGR
1454 "pand %%mm7, %%mm0 \n\t"
1455 "pand %%mm5, %%mm1 \n\t"
1456 "pand %%mm6, %%mm2 \n\t"
1457 "por %%mm0, %%mm1 \n\t"
1458 "por %%mm2, %%mm1 \n\t"
1459 "movq 14(%1, %%"REG_a"), %%mm0 \n\t" // R BGR BGR B
1460 MOVNTQ" %%mm1, 8(%2, %%"REG_a")\n\t" // B RGB RGB R
1461 "movq 16(%1, %%"REG_a"), %%mm1 \n\t" // GR BGR BGR
1462 "movq 18(%1, %%"REG_a"), %%mm2 \n\t" // BGR BGR BG
1463 "pand %%mm6, %%mm0 \n\t"
1464 "pand %%mm7, %%mm1 \n\t"
1465 "pand %%mm5, %%mm2 \n\t"
1466 "por %%mm0, %%mm1 \n\t"
1467 "por %%mm2, %%mm1 \n\t"
1468 MOVNTQ" %%mm1, 16(%2, %%"REG_a")\n\t"
1469 "add $24, %%"REG_a" \n\t"
1472 : "r" (src-mmx_size), "r"(dst-mmx_size)
1475 __asm __volatile(SFENCE:::"memory");
1476 __asm __volatile(EMMS:::"memory");
1478 if(mmx_size==23) return; //finihsed, was multiple of 8
1482 src_size= 23-mmx_size;
1486 for(i=0; i<src_size; i+=3)
1490 dst[i + 1] = src[i + 1];
1491 dst[i + 2] = src[i + 0];
1496 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1497 long width, long height,
1498 long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1501 const long chromWidth= width>>1;
1502 for(y=0; y<height; y++)
1505 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1507 "xor %%"REG_a", %%"REG_a" \n\t"
1510 PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1511 PREFETCH" 32(%2, %%"REG_a") \n\t"
1512 PREFETCH" 32(%3, %%"REG_a") \n\t"
1513 "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1514 "movq %%mm0, %%mm2 \n\t" // U(0)
1515 "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1516 "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1517 "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1519 "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1520 "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1521 "movq %%mm3, %%mm4 \n\t" // Y(0)
1522 "movq %%mm5, %%mm6 \n\t" // Y(8)
1523 "punpcklbw %%mm0, %%mm3 \n\t" // YUYV YUYV(0)
1524 "punpckhbw %%mm0, %%mm4 \n\t" // YUYV YUYV(4)
1525 "punpcklbw %%mm2, %%mm5 \n\t" // YUYV YUYV(8)
1526 "punpckhbw %%mm2, %%mm6 \n\t" // YUYV YUYV(12)
1528 MOVNTQ" %%mm3, (%0, %%"REG_a", 4)\n\t"
1529 MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1530 MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4)\n\t"
1531 MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1533 "add $8, %%"REG_a" \n\t"
1534 "cmp %4, %%"REG_a" \n\t"
1536 ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1541 #if defined ARCH_ALPHA && defined HAVE_MVI
1542 #define pl2yuy2(n) \
1547 asm("unpkbw %1, %0" : "=r"(y1) : "r"(y1)); \
1548 asm("unpkbw %1, %0" : "=r"(y2) : "r"(y2)); \
1549 asm("unpkbl %1, %0" : "=r"(u) : "r"(u)); \
1550 asm("unpkbl %1, %0" : "=r"(v) : "r"(v)); \
1551 yuv1 = (u << 8) + (v << 24); \
1558 uint64_t *qdst = (uint64_t *) dst;
1559 uint64_t *qdst2 = (uint64_t *) (dst + dstStride);
1560 const uint32_t *yc = (uint32_t *) ysrc;
1561 const uint32_t *yc2 = (uint32_t *) (ysrc + lumStride);
1562 const uint16_t *uc = (uint16_t*) usrc, *vc = (uint16_t*) vsrc;
1563 for(i = 0; i < chromWidth; i += 8){
1564 uint64_t y1, y2, yuv1, yuv2;
1567 asm("ldq $31,64(%0)" :: "r"(yc));
1568 asm("ldq $31,64(%0)" :: "r"(yc2));
1569 asm("ldq $31,64(%0)" :: "r"(uc));
1570 asm("ldq $31,64(%0)" :: "r"(vc));
1588 #elif __WORDSIZE >= 64
1590 uint64_t *ldst = (uint64_t *) dst;
1591 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1592 for(i = 0; i < chromWidth; i += 2){
1594 k = yc[0] + (uc[0] << 8) +
1595 (yc[1] << 16) + (vc[0] << 24);
1596 l = yc[2] + (uc[1] << 8) +
1597 (yc[3] << 16) + (vc[1] << 24);
1598 *ldst++ = k + (l << 32);
1605 int i, *idst = (int32_t *) dst;
1606 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1607 for(i = 0; i < chromWidth; i++){
1608 #ifdef WORDS_BIGENDIAN
1609 *idst++ = (yc[0] << 24)+ (uc[0] << 16) +
1610 (yc[1] << 8) + (vc[0] << 0);
1612 *idst++ = yc[0] + (uc[0] << 8) +
1613 (yc[1] << 16) + (vc[0] << 24);
1621 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1623 usrc += chromStride;
1624 vsrc += chromStride;
1638 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1639 * problem for anyone then tell me, and ill fix it)
1641 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1642 long width, long height,
1643 long lumStride, long chromStride, long dstStride)
1645 //FIXME interpolate chroma
1646 RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1649 static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1650 long width, long height,
1651 long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1654 const long chromWidth= width>>1;
1655 for(y=0; y<height; y++)
1658 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1660 "xor %%"REG_a", %%"REG_a" \n\t"
1663 PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1664 PREFETCH" 32(%2, %%"REG_a") \n\t"
1665 PREFETCH" 32(%3, %%"REG_a") \n\t"
1666 "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1667 "movq %%mm0, %%mm2 \n\t" // U(0)
1668 "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1669 "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1670 "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1672 "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1673 "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1674 "movq %%mm0, %%mm4 \n\t" // Y(0)
1675 "movq %%mm2, %%mm6 \n\t" // Y(8)
1676 "punpcklbw %%mm3, %%mm0 \n\t" // YUYV YUYV(0)
1677 "punpckhbw %%mm3, %%mm4 \n\t" // YUYV YUYV(4)
1678 "punpcklbw %%mm5, %%mm2 \n\t" // YUYV YUYV(8)
1679 "punpckhbw %%mm5, %%mm6 \n\t" // YUYV YUYV(12)
1681 MOVNTQ" %%mm0, (%0, %%"REG_a", 4)\n\t"
1682 MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1683 MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4)\n\t"
1684 MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1686 "add $8, %%"REG_a" \n\t"
1687 "cmp %4, %%"REG_a" \n\t"
1689 ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1693 //FIXME adapt the alpha asm code from yv12->yuy2
1695 #if __WORDSIZE >= 64
1697 uint64_t *ldst = (uint64_t *) dst;
1698 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1699 for(i = 0; i < chromWidth; i += 2){
1701 k = uc[0] + (yc[0] << 8) +
1702 (vc[0] << 16) + (yc[1] << 24);
1703 l = uc[1] + (yc[2] << 8) +
1704 (vc[1] << 16) + (yc[3] << 24);
1705 *ldst++ = k + (l << 32);
1712 int i, *idst = (int32_t *) dst;
1713 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1714 for(i = 0; i < chromWidth; i++){
1715 #ifdef WORDS_BIGENDIAN
1716 *idst++ = (uc[0] << 24)+ (yc[0] << 16) +
1717 (vc[0] << 8) + (yc[1] << 0);
1719 *idst++ = uc[0] + (yc[0] << 8) +
1720 (vc[0] << 16) + (yc[1] << 24);
1728 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1730 usrc += chromStride;
1731 vsrc += chromStride;
1745 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1746 * problem for anyone then tell me, and ill fix it)
1748 static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1749 long width, long height,
1750 long lumStride, long chromStride, long dstStride)
1752 //FIXME interpolate chroma
1753 RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1758 * width should be a multiple of 16
1760 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1761 long width, long height,
1762 long lumStride, long chromStride, long dstStride)
1764 RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1769 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1770 * problem for anyone then tell me, and ill fix it)
1772 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1773 long width, long height,
1774 long lumStride, long chromStride, long srcStride)
1777 const long chromWidth= width>>1;
1778 for(y=0; y<height; y+=2)
1782 "xor %%"REG_a", %%"REG_a" \n\t"
1783 "pcmpeqw %%mm7, %%mm7 \n\t"
1784 "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
1787 PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1788 "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1789 "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1790 "movq %%mm0, %%mm2 \n\t" // YUYV YUYV(0)
1791 "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(4)
1792 "psrlw $8, %%mm0 \n\t" // U0V0 U0V0(0)
1793 "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(4)
1794 "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
1795 "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
1796 "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1797 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
1799 MOVNTQ" %%mm2, (%1, %%"REG_a", 2)\n\t"
1801 "movq 16(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(8)
1802 "movq 24(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(12)
1803 "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(8)
1804 "movq %%mm2, %%mm4 \n\t" // YUYV YUYV(12)
1805 "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(8)
1806 "psrlw $8, %%mm2 \n\t" // U0V0 U0V0(12)
1807 "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
1808 "pand %%mm7, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
1809 "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
1810 "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
1812 MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2)\n\t"
1814 "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
1815 "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
1816 "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
1817 "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
1818 "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
1819 "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
1820 "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
1821 "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
1823 MOVNTQ" %%mm0, (%3, %%"REG_a") \n\t"
1824 MOVNTQ" %%mm2, (%2, %%"REG_a") \n\t"
1826 "add $8, %%"REG_a" \n\t"
1827 "cmp %4, %%"REG_a" \n\t"
1829 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1830 : "memory", "%"REG_a
1837 "xor %%"REG_a", %%"REG_a" \n\t"
1840 PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1841 "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1842 "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1843 "movq 16(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(8)
1844 "movq 24(%0, %%"REG_a", 4), %%mm3\n\t" // YUYV YUYV(12)
1845 "pand %%mm7, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
1846 "pand %%mm7, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
1847 "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
1848 "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
1849 "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
1850 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
1852 MOVNTQ" %%mm0, (%1, %%"REG_a", 2)\n\t"
1853 MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2)\n\t"
1855 "add $8, %%"REG_a" \n\t"
1856 "cmp %4, %%"REG_a" \n\t"
1859 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1860 : "memory", "%"REG_a
1864 for(i=0; i<chromWidth; i++)
1866 ydst[2*i+0] = src[4*i+0];
1867 udst[i] = src[4*i+1];
1868 ydst[2*i+1] = src[4*i+2];
1869 vdst[i] = src[4*i+3];
1874 for(i=0; i<chromWidth; i++)
1876 ydst[2*i+0] = src[4*i+0];
1877 ydst[2*i+1] = src[4*i+2];
1880 udst += chromStride;
1881 vdst += chromStride;
1886 asm volatile( EMMS" \n\t"
1892 static inline void RENAME(yvu9toyv12)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,
1893 uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1894 long width, long height, long lumStride, long chromStride)
1897 memcpy(ydst, ysrc, width*height);
1899 /* XXX: implement upscaling for U,V */
1902 static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, long srcWidth, long srcHeight, long srcStride, long dstStride)
1909 for(x=0; x<srcWidth-1; x++){
1910 dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1911 dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1913 dst[2*srcWidth-1]= src[srcWidth-1];
1917 for(y=1; y<srcHeight; y++){
1918 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
1919 const long mmxSize= srcWidth&~15;
1921 "mov %4, %%"REG_a" \n\t"
1923 "movq (%0, %%"REG_a"), %%mm0 \n\t"
1924 "movq (%1, %%"REG_a"), %%mm1 \n\t"
1925 "movq 1(%0, %%"REG_a"), %%mm2 \n\t"
1926 "movq 1(%1, %%"REG_a"), %%mm3 \n\t"
1927 "movq -1(%0, %%"REG_a"), %%mm4 \n\t"
1928 "movq -1(%1, %%"REG_a"), %%mm5 \n\t"
1929 PAVGB" %%mm0, %%mm5 \n\t"
1930 PAVGB" %%mm0, %%mm3 \n\t"
1931 PAVGB" %%mm0, %%mm5 \n\t"
1932 PAVGB" %%mm0, %%mm3 \n\t"
1933 PAVGB" %%mm1, %%mm4 \n\t"
1934 PAVGB" %%mm1, %%mm2 \n\t"
1935 PAVGB" %%mm1, %%mm4 \n\t"
1936 PAVGB" %%mm1, %%mm2 \n\t"
1937 "movq %%mm5, %%mm7 \n\t"
1938 "movq %%mm4, %%mm6 \n\t"
1939 "punpcklbw %%mm3, %%mm5 \n\t"
1940 "punpckhbw %%mm3, %%mm7 \n\t"
1941 "punpcklbw %%mm2, %%mm4 \n\t"
1942 "punpckhbw %%mm2, %%mm6 \n\t"
1944 MOVNTQ" %%mm5, (%2, %%"REG_a", 2)\n\t"
1945 MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1946 MOVNTQ" %%mm4, (%3, %%"REG_a", 2)\n\t"
1947 MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1949 "movq %%mm5, (%2, %%"REG_a", 2) \n\t"
1950 "movq %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1951 "movq %%mm4, (%3, %%"REG_a", 2) \n\t"
1952 "movq %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1954 "add $8, %%"REG_a" \n\t"
1956 :: "r" (src + mmxSize ), "r" (src + srcStride + mmxSize ),
1957 "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
1963 const long mmxSize=1;
1965 dst[0 ]= (3*src[0] + src[srcStride])>>2;
1966 dst[dstStride]= ( src[0] + 3*src[srcStride])>>2;
1968 for(x=mmxSize-1; x<srcWidth-1; x++){
1969 dst[2*x +1]= (3*src[x+0] + src[x+srcStride+1])>>2;
1970 dst[2*x+dstStride+2]= ( src[x+0] + 3*src[x+srcStride+1])>>2;
1971 dst[2*x+dstStride+1]= ( src[x+1] + 3*src[x+srcStride ])>>2;
1972 dst[2*x +2]= (3*src[x+1] + src[x+srcStride ])>>2;
1974 dst[srcWidth*2 -1 ]= (3*src[srcWidth-1] + src[srcWidth-1 + srcStride])>>2;
1975 dst[srcWidth*2 -1 + dstStride]= ( src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
1985 for(x=0; x<srcWidth-1; x++){
1986 dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1987 dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1989 dst[2*srcWidth-1]= src[srcWidth-1];
1991 for(x=0; x<srcWidth; x++){
1998 asm volatile( EMMS" \n\t"
2006 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
2007 * problem for anyone then tell me, and ill fix it)
2008 * chrominance data is only taken from every secound line others are ignored FIXME write HQ version
2010 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2011 long width, long height,
2012 long lumStride, long chromStride, long srcStride)
2015 const long chromWidth= width>>1;
2016 for(y=0; y<height; y+=2)
2020 "xorl %%eax, %%eax \n\t"
2021 "pcmpeqw %%mm7, %%mm7 \n\t"
2022 "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
2025 PREFETCH" 64(%0, %%eax, 4) \n\t"
2026 "movq (%0, %%eax, 4), %%mm0 \n\t" // UYVY UYVY(0)
2027 "movq 8(%0, %%eax, 4), %%mm1 \n\t" // UYVY UYVY(4)
2028 "movq %%mm0, %%mm2 \n\t" // UYVY UYVY(0)
2029 "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(4)
2030 "pand %%mm7, %%mm0 \n\t" // U0V0 U0V0(0)
2031 "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(4)
2032 "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
2033 "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
2034 "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
2035 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
2037 MOVNTQ" %%mm2, (%1, %%eax, 2) \n\t"
2039 "movq 16(%0, %%eax, 4), %%mm1 \n\t" // UYVY UYVY(8)
2040 "movq 24(%0, %%eax, 4), %%mm2 \n\t" // UYVY UYVY(12)
2041 "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(8)
2042 "movq %%mm2, %%mm4 \n\t" // UYVY UYVY(12)
2043 "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(8)
2044 "pand %%mm7, %%mm2 \n\t" // U0V0 U0V0(12)
2045 "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
2046 "psrlw $8, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
2047 "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
2048 "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
2050 MOVNTQ" %%mm3, 8(%1, %%eax, 2) \n\t"
2052 "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
2053 "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
2054 "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
2055 "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
2056 "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
2057 "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
2058 "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
2059 "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
2061 MOVNTQ" %%mm0, (%3, %%eax) \n\t"
2062 MOVNTQ" %%mm2, (%2, %%eax) \n\t"
2064 "addl $8, %%eax \n\t"
2065 "cmpl %4, %%eax \n\t"
2067 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2075 "xorl %%eax, %%eax \n\t"
2078 PREFETCH" 64(%0, %%eax, 4) \n\t"
2079 "movq (%0, %%eax, 4), %%mm0 \n\t" // YUYV YUYV(0)
2080 "movq 8(%0, %%eax, 4), %%mm1 \n\t" // YUYV YUYV(4)
2081 "movq 16(%0, %%eax, 4), %%mm2 \n\t" // YUYV YUYV(8)
2082 "movq 24(%0, %%eax, 4), %%mm3 \n\t" // YUYV YUYV(12)
2083 "psrlw $8, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
2084 "psrlw $8, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
2085 "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
2086 "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
2087 "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
2088 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
2090 MOVNTQ" %%mm0, (%1, %%eax, 2) \n\t"
2091 MOVNTQ" %%mm2, 8(%1, %%eax, 2) \n\t"
2093 "addl $8, %%eax \n\t"
2094 "cmpl %4, %%eax \n\t"
2097 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2102 for(i=0; i<chromWidth; i++)
2104 udst[i] = src[4*i+0];
2105 ydst[2*i+0] = src[4*i+1];
2106 vdst[i] = src[4*i+2];
2107 ydst[2*i+1] = src[4*i+3];
2112 for(i=0; i<chromWidth; i++)
2114 ydst[2*i+0] = src[4*i+1];
2115 ydst[2*i+1] = src[4*i+3];
2118 udst += chromStride;
2119 vdst += chromStride;
2124 asm volatile( EMMS" \n\t"
2132 * height should be a multiple of 2 and width should be a multiple of 2 (if this is a
2133 * problem for anyone then tell me, and ill fix it)
2134 * chrominance data is only taken from every secound line others are ignored in the C version FIXME write HQ version
2136 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2137 long width, long height,
2138 long lumStride, long chromStride, long srcStride)
2141 const long chromWidth= width>>1;
2143 for(y=0; y<height-2; y+=2)
2149 "mov %2, %%"REG_a" \n\t"
2150 "movq "MANGLE(bgr2YCoeff)", %%mm6 \n\t"
2151 "movq "MANGLE(w1111)", %%mm5 \n\t"
2152 "pxor %%mm7, %%mm7 \n\t"
2153 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2156 PREFETCH" 64(%0, %%"REG_d") \n\t"
2157 "movd (%0, %%"REG_d"), %%mm0 \n\t"
2158 "movd 3(%0, %%"REG_d"), %%mm1 \n\t"
2159 "punpcklbw %%mm7, %%mm0 \n\t"
2160 "punpcklbw %%mm7, %%mm1 \n\t"
2161 "movd 6(%0, %%"REG_d"), %%mm2 \n\t"
2162 "movd 9(%0, %%"REG_d"), %%mm3 \n\t"
2163 "punpcklbw %%mm7, %%mm2 \n\t"
2164 "punpcklbw %%mm7, %%mm3 \n\t"
2165 "pmaddwd %%mm6, %%mm0 \n\t"
2166 "pmaddwd %%mm6, %%mm1 \n\t"
2167 "pmaddwd %%mm6, %%mm2 \n\t"
2168 "pmaddwd %%mm6, %%mm3 \n\t"
2169 #ifndef FAST_BGR2YV12
2170 "psrad $8, %%mm0 \n\t"
2171 "psrad $8, %%mm1 \n\t"
2172 "psrad $8, %%mm2 \n\t"
2173 "psrad $8, %%mm3 \n\t"
2175 "packssdw %%mm1, %%mm0 \n\t"
2176 "packssdw %%mm3, %%mm2 \n\t"
2177 "pmaddwd %%mm5, %%mm0 \n\t"
2178 "pmaddwd %%mm5, %%mm2 \n\t"
2179 "packssdw %%mm2, %%mm0 \n\t"
2180 "psraw $7, %%mm0 \n\t"
2182 "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
2183 "movd 15(%0, %%"REG_d"), %%mm1 \n\t"
2184 "punpcklbw %%mm7, %%mm4 \n\t"
2185 "punpcklbw %%mm7, %%mm1 \n\t"
2186 "movd 18(%0, %%"REG_d"), %%mm2 \n\t"
2187 "movd 21(%0, %%"REG_d"), %%mm3 \n\t"
2188 "punpcklbw %%mm7, %%mm2 \n\t"
2189 "punpcklbw %%mm7, %%mm3 \n\t"
2190 "pmaddwd %%mm6, %%mm4 \n\t"
2191 "pmaddwd %%mm6, %%mm1 \n\t"
2192 "pmaddwd %%mm6, %%mm2 \n\t"
2193 "pmaddwd %%mm6, %%mm3 \n\t"
2194 #ifndef FAST_BGR2YV12
2195 "psrad $8, %%mm4 \n\t"
2196 "psrad $8, %%mm1 \n\t"
2197 "psrad $8, %%mm2 \n\t"
2198 "psrad $8, %%mm3 \n\t"
2200 "packssdw %%mm1, %%mm4 \n\t"
2201 "packssdw %%mm3, %%mm2 \n\t"
2202 "pmaddwd %%mm5, %%mm4 \n\t"
2203 "pmaddwd %%mm5, %%mm2 \n\t"
2204 "add $24, %%"REG_d" \n\t"
2205 "packssdw %%mm2, %%mm4 \n\t"
2206 "psraw $7, %%mm4 \n\t"
2208 "packuswb %%mm4, %%mm0 \n\t"
2209 "paddusb "MANGLE(bgr2YOffset)", %%mm0 \n\t"
2211 MOVNTQ" %%mm0, (%1, %%"REG_a") \n\t"
2212 "add $8, %%"REG_a" \n\t"
2214 : : "r" (src+width*3), "r" (ydst+width), "g" (-width)
2215 : "%"REG_a, "%"REG_d
2222 "mov %4, %%"REG_a" \n\t"
2223 "movq "MANGLE(w1111)", %%mm5 \n\t"
2224 "movq "MANGLE(bgr2UCoeff)", %%mm6 \n\t"
2225 "pxor %%mm7, %%mm7 \n\t"
2226 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2227 "add %%"REG_d", %%"REG_d" \n\t"
2230 PREFETCH" 64(%0, %%"REG_d") \n\t"
2231 PREFETCH" 64(%1, %%"REG_d") \n\t"
2232 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2233 "movq (%0, %%"REG_d"), %%mm0 \n\t"
2234 "movq (%1, %%"REG_d"), %%mm1 \n\t"
2235 "movq 6(%0, %%"REG_d"), %%mm2 \n\t"
2236 "movq 6(%1, %%"REG_d"), %%mm3 \n\t"
2237 PAVGB" %%mm1, %%mm0 \n\t"
2238 PAVGB" %%mm3, %%mm2 \n\t"
2239 "movq %%mm0, %%mm1 \n\t"
2240 "movq %%mm2, %%mm3 \n\t"
2241 "psrlq $24, %%mm0 \n\t"
2242 "psrlq $24, %%mm2 \n\t"
2243 PAVGB" %%mm1, %%mm0 \n\t"
2244 PAVGB" %%mm3, %%mm2 \n\t"
2245 "punpcklbw %%mm7, %%mm0 \n\t"
2246 "punpcklbw %%mm7, %%mm2 \n\t"
2248 "movd (%0, %%"REG_d"), %%mm0 \n\t"
2249 "movd (%1, %%"REG_d"), %%mm1 \n\t"
2250 "movd 3(%0, %%"REG_d"), %%mm2 \n\t"
2251 "movd 3(%1, %%"REG_d"), %%mm3 \n\t"
2252 "punpcklbw %%mm7, %%mm0 \n\t"
2253 "punpcklbw %%mm7, %%mm1 \n\t"
2254 "punpcklbw %%mm7, %%mm2 \n\t"
2255 "punpcklbw %%mm7, %%mm3 \n\t"
2256 "paddw %%mm1, %%mm0 \n\t"
2257 "paddw %%mm3, %%mm2 \n\t"
2258 "paddw %%mm2, %%mm0 \n\t"
2259 "movd 6(%0, %%"REG_d"), %%mm4 \n\t"
2260 "movd 6(%1, %%"REG_d"), %%mm1 \n\t"
2261 "movd 9(%0, %%"REG_d"), %%mm2 \n\t"
2262 "movd 9(%1, %%"REG_d"), %%mm3 \n\t"
2263 "punpcklbw %%mm7, %%mm4 \n\t"
2264 "punpcklbw %%mm7, %%mm1 \n\t"
2265 "punpcklbw %%mm7, %%mm2 \n\t"
2266 "punpcklbw %%mm7, %%mm3 \n\t"
2267 "paddw %%mm1, %%mm4 \n\t"
2268 "paddw %%mm3, %%mm2 \n\t"
2269 "paddw %%mm4, %%mm2 \n\t"
2270 "psrlw $2, %%mm0 \n\t"
2271 "psrlw $2, %%mm2 \n\t"
2273 "movq "MANGLE(bgr2VCoeff)", %%mm1 \n\t"
2274 "movq "MANGLE(bgr2VCoeff)", %%mm3 \n\t"
2276 "pmaddwd %%mm0, %%mm1 \n\t"
2277 "pmaddwd %%mm2, %%mm3 \n\t"
2278 "pmaddwd %%mm6, %%mm0 \n\t"
2279 "pmaddwd %%mm6, %%mm2 \n\t"
2280 #ifndef FAST_BGR2YV12
2281 "psrad $8, %%mm0 \n\t"
2282 "psrad $8, %%mm1 \n\t"
2283 "psrad $8, %%mm2 \n\t"
2284 "psrad $8, %%mm3 \n\t"
2286 "packssdw %%mm2, %%mm0 \n\t"
2287 "packssdw %%mm3, %%mm1 \n\t"
2288 "pmaddwd %%mm5, %%mm0 \n\t"
2289 "pmaddwd %%mm5, %%mm1 \n\t"
2290 "packssdw %%mm1, %%mm0 \n\t" // V1 V0 U1 U0
2291 "psraw $7, %%mm0 \n\t"
2293 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2294 "movq 12(%0, %%"REG_d"), %%mm4 \n\t"
2295 "movq 12(%1, %%"REG_d"), %%mm1 \n\t"
2296 "movq 18(%0, %%"REG_d"), %%mm2 \n\t"
2297 "movq 18(%1, %%"REG_d"), %%mm3 \n\t"
2298 PAVGB" %%mm1, %%mm4 \n\t"
2299 PAVGB" %%mm3, %%mm2 \n\t"
2300 "movq %%mm4, %%mm1 \n\t"
2301 "movq %%mm2, %%mm3 \n\t"
2302 "psrlq $24, %%mm4 \n\t"
2303 "psrlq $24, %%mm2 \n\t"
2304 PAVGB" %%mm1, %%mm4 \n\t"
2305 PAVGB" %%mm3, %%mm2 \n\t"
2306 "punpcklbw %%mm7, %%mm4 \n\t"
2307 "punpcklbw %%mm7, %%mm2 \n\t"
2309 "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
2310 "movd 12(%1, %%"REG_d"), %%mm1 \n\t"
2311 "movd 15(%0, %%"REG_d"), %%mm2 \n\t"
2312 "movd 15(%1, %%"REG_d"), %%mm3 \n\t"
2313 "punpcklbw %%mm7, %%mm4 \n\t"
2314 "punpcklbw %%mm7, %%mm1 \n\t"
2315 "punpcklbw %%mm7, %%mm2 \n\t"
2316 "punpcklbw %%mm7, %%mm3 \n\t"
2317 "paddw %%mm1, %%mm4 \n\t"
2318 "paddw %%mm3, %%mm2 \n\t"
2319 "paddw %%mm2, %%mm4 \n\t"
2320 "movd 18(%0, %%"REG_d"), %%mm5 \n\t"
2321 "movd 18(%1, %%"REG_d"), %%mm1 \n\t"
2322 "movd 21(%0, %%"REG_d"), %%mm2 \n\t"
2323 "movd 21(%1, %%"REG_d"), %%mm3 \n\t"
2324 "punpcklbw %%mm7, %%mm5 \n\t"
2325 "punpcklbw %%mm7, %%mm1 \n\t"
2326 "punpcklbw %%mm7, %%mm2 \n\t"
2327 "punpcklbw %%mm7, %%mm3 \n\t"
2328 "paddw %%mm1, %%mm5 \n\t"
2329 "paddw %%mm3, %%mm2 \n\t"
2330 "paddw %%mm5, %%mm2 \n\t"
2331 "movq "MANGLE(w1111)", %%mm5 \n\t"
2332 "psrlw $2, %%mm4 \n\t"
2333 "psrlw $2, %%mm2 \n\t"
2335 "movq "MANGLE(bgr2VCoeff)", %%mm1 \n\t"
2336 "movq "MANGLE(bgr2VCoeff)", %%mm3 \n\t"
2338 "pmaddwd %%mm4, %%mm1 \n\t"
2339 "pmaddwd %%mm2, %%mm3 \n\t"
2340 "pmaddwd %%mm6, %%mm4 \n\t"
2341 "pmaddwd %%mm6, %%mm2 \n\t"
2342 #ifndef FAST_BGR2YV12
2343 "psrad $8, %%mm4 \n\t"
2344 "psrad $8, %%mm1 \n\t"
2345 "psrad $8, %%mm2 \n\t"
2346 "psrad $8, %%mm3 \n\t"
2348 "packssdw %%mm2, %%mm4 \n\t"
2349 "packssdw %%mm3, %%mm1 \n\t"
2350 "pmaddwd %%mm5, %%mm4 \n\t"
2351 "pmaddwd %%mm5, %%mm1 \n\t"
2352 "add $24, %%"REG_d" \n\t"
2353 "packssdw %%mm1, %%mm4 \n\t" // V3 V2 U3 U2
2354 "psraw $7, %%mm4 \n\t"
2356 "movq %%mm0, %%mm1 \n\t"
2357 "punpckldq %%mm4, %%mm0 \n\t"
2358 "punpckhdq %%mm4, %%mm1 \n\t"
2359 "packsswb %%mm1, %%mm0 \n\t"
2360 "paddb "MANGLE(bgr2UVOffset)", %%mm0 \n\t"
2361 "movd %%mm0, (%2, %%"REG_a") \n\t"
2362 "punpckhdq %%mm0, %%mm0 \n\t"
2363 "movd %%mm0, (%3, %%"REG_a") \n\t"
2364 "add $4, %%"REG_a" \n\t"
2366 : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth)
2367 : "%"REG_a, "%"REG_d
2370 udst += chromStride;
2371 vdst += chromStride;
2375 asm volatile( EMMS" \n\t"
2381 for(; y<height; y+=2)
2384 for(i=0; i<chromWidth; i++)
2386 unsigned int b= src[6*i+0];
2387 unsigned int g= src[6*i+1];
2388 unsigned int r= src[6*i+2];
2390 unsigned int Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2391 unsigned int V = ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128;
2392 unsigned int U = ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128;
2402 Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2408 for(i=0; i<chromWidth; i++)
2410 unsigned int b= src[6*i+0];
2411 unsigned int g= src[6*i+1];
2412 unsigned int r= src[6*i+2];
2414 unsigned int Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2422 Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2425 udst += chromStride;
2426 vdst += chromStride;
2432 void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest,
2433 long width, long height, long src1Stride,
2434 long src2Stride, long dstStride){
2437 for(h=0; h < height; h++)
2444 "xor %%"REG_a", %%"REG_a" \n\t"
2446 PREFETCH" 64(%1, %%"REG_a") \n\t"
2447 PREFETCH" 64(%2, %%"REG_a") \n\t"
2448 "movdqa (%1, %%"REG_a"), %%xmm0 \n\t"
2449 "movdqa (%1, %%"REG_a"), %%xmm1 \n\t"
2450 "movdqa (%2, %%"REG_a"), %%xmm2 \n\t"
2451 "punpcklbw %%xmm2, %%xmm0 \n\t"
2452 "punpckhbw %%xmm2, %%xmm1 \n\t"
2453 "movntdq %%xmm0, (%0, %%"REG_a", 2)\n\t"
2454 "movntdq %%xmm1, 16(%0, %%"REG_a", 2)\n\t"
2455 "add $16, %%"REG_a" \n\t"
2456 "cmp %3, %%"REG_a" \n\t"
2458 ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2459 : "memory", "%"REG_a""
2463 "xor %%"REG_a", %%"REG_a" \n\t"
2465 PREFETCH" 64(%1, %%"REG_a") \n\t"
2466 PREFETCH" 64(%2, %%"REG_a") \n\t"
2467 "movq (%1, %%"REG_a"), %%mm0 \n\t"
2468 "movq 8(%1, %%"REG_a"), %%mm2 \n\t"
2469 "movq %%mm0, %%mm1 \n\t"
2470 "movq %%mm2, %%mm3 \n\t"
2471 "movq (%2, %%"REG_a"), %%mm4 \n\t"
2472 "movq 8(%2, %%"REG_a"), %%mm5 \n\t"
2473 "punpcklbw %%mm4, %%mm0 \n\t"
2474 "punpckhbw %%mm4, %%mm1 \n\t"
2475 "punpcklbw %%mm5, %%mm2 \n\t"
2476 "punpckhbw %%mm5, %%mm3 \n\t"
2477 MOVNTQ" %%mm0, (%0, %%"REG_a", 2)\n\t"
2478 MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2)\n\t"
2479 MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2)\n\t"
2480 MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2)\n\t"
2481 "add $16, %%"REG_a" \n\t"
2482 "cmp %3, %%"REG_a" \n\t"
2484 ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2485 : "memory", "%"REG_a
2488 for(w= (width&(~15)); w < width; w++)
2490 dest[2*w+0] = src1[w];
2491 dest[2*w+1] = src2[w];
2494 for(w=0; w < width; w++)
2496 dest[2*w+0] = src1[w];
2497 dest[2*w+1] = src2[w];
2513 static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
2514 uint8_t *dst1, uint8_t *dst2,
2515 long width, long height,
2516 long srcStride1, long srcStride2,
2517 long dstStride1, long dstStride2)
2520 w=width/2; h=height/2;
2525 ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
2528 const uint8_t* s1=src1+srcStride1*(y>>1);
2529 uint8_t* d=dst1+dstStride1*y;
2536 "movq %1, %%mm0\n\t"
2537 "movq 8%1, %%mm2\n\t"
2538 "movq 16%1, %%mm4\n\t"
2539 "movq 24%1, %%mm6\n\t"
2540 "movq %%mm0, %%mm1\n\t"
2541 "movq %%mm2, %%mm3\n\t"
2542 "movq %%mm4, %%mm5\n\t"
2543 "movq %%mm6, %%mm7\n\t"
2544 "punpcklbw %%mm0, %%mm0\n\t"
2545 "punpckhbw %%mm1, %%mm1\n\t"
2546 "punpcklbw %%mm2, %%mm2\n\t"
2547 "punpckhbw %%mm3, %%mm3\n\t"
2548 "punpcklbw %%mm4, %%mm4\n\t"
2549 "punpckhbw %%mm5, %%mm5\n\t"
2550 "punpcklbw %%mm6, %%mm6\n\t"
2551 "punpckhbw %%mm7, %%mm7\n\t"
2552 MOVNTQ" %%mm0, %0\n\t"
2553 MOVNTQ" %%mm1, 8%0\n\t"
2554 MOVNTQ" %%mm2, 16%0\n\t"
2555 MOVNTQ" %%mm3, 24%0\n\t"
2556 MOVNTQ" %%mm4, 32%0\n\t"
2557 MOVNTQ" %%mm5, 40%0\n\t"
2558 MOVNTQ" %%mm6, 48%0\n\t"
2559 MOVNTQ" %%mm7, 56%0"
2565 for(;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
2568 const uint8_t* s2=src2+srcStride2*(y>>1);
2569 uint8_t* d=dst2+dstStride2*y;
2576 "movq %1, %%mm0\n\t"
2577 "movq 8%1, %%mm2\n\t"
2578 "movq 16%1, %%mm4\n\t"
2579 "movq 24%1, %%mm6\n\t"
2580 "movq %%mm0, %%mm1\n\t"
2581 "movq %%mm2, %%mm3\n\t"
2582 "movq %%mm4, %%mm5\n\t"
2583 "movq %%mm6, %%mm7\n\t"
2584 "punpcklbw %%mm0, %%mm0\n\t"
2585 "punpckhbw %%mm1, %%mm1\n\t"
2586 "punpcklbw %%mm2, %%mm2\n\t"
2587 "punpckhbw %%mm3, %%mm3\n\t"
2588 "punpcklbw %%mm4, %%mm4\n\t"
2589 "punpckhbw %%mm5, %%mm5\n\t"
2590 "punpcklbw %%mm6, %%mm6\n\t"
2591 "punpckhbw %%mm7, %%mm7\n\t"
2592 MOVNTQ" %%mm0, %0\n\t"
2593 MOVNTQ" %%mm1, 8%0\n\t"
2594 MOVNTQ" %%mm2, 16%0\n\t"
2595 MOVNTQ" %%mm3, 24%0\n\t"
2596 MOVNTQ" %%mm4, 32%0\n\t"
2597 MOVNTQ" %%mm5, 40%0\n\t"
2598 MOVNTQ" %%mm6, 48%0\n\t"
2599 MOVNTQ" %%mm7, 56%0"
2605 for(;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
2616 static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
2618 long width, long height,
2619 long srcStride1, long srcStride2,
2620 long srcStride3, long dstStride)
2623 w=width/2; h=height;
2625 const uint8_t* yp=src1+srcStride1*y;
2626 const uint8_t* up=src2+srcStride2*(y>>2);
2627 const uint8_t* vp=src3+srcStride3*(y>>2);
2628 uint8_t* d=dst+dstStride*y;
2634 PREFETCH" 32(%1, %0)\n\t"
2635 PREFETCH" 32(%2, %0)\n\t"
2636 PREFETCH" 32(%3, %0)\n\t"
2637 "movq (%1, %0, 4), %%mm0\n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2638 "movq (%2, %0), %%mm1\n\t" /* U0U1U2U3U4U5U6U7 */
2639 "movq (%3, %0), %%mm2\n\t" /* V0V1V2V3V4V5V6V7 */
2640 "movq %%mm0, %%mm3\n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2641 "movq %%mm1, %%mm4\n\t" /* U0U1U2U3U4U5U6U7 */
2642 "movq %%mm2, %%mm5\n\t" /* V0V1V2V3V4V5V6V7 */
2643 "punpcklbw %%mm1, %%mm1\n\t" /* U0U0 U1U1 U2U2 U3U3 */
2644 "punpcklbw %%mm2, %%mm2\n\t" /* V0V0 V1V1 V2V2 V3V3 */
2645 "punpckhbw %%mm4, %%mm4\n\t" /* U4U4 U5U5 U6U6 U7U7 */
2646 "punpckhbw %%mm5, %%mm5\n\t" /* V4V4 V5V5 V6V6 V7V7 */
2648 "movq %%mm1, %%mm6\n\t"
2649 "punpcklbw %%mm2, %%mm1\n\t" /* U0V0 U0V0 U1V1 U1V1*/
2650 "punpcklbw %%mm1, %%mm0\n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
2651 "punpckhbw %%mm1, %%mm3\n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
2652 MOVNTQ" %%mm0, (%4, %0, 8)\n\t"
2653 MOVNTQ" %%mm3, 8(%4, %0, 8)\n\t"
2655 "punpckhbw %%mm2, %%mm6\n\t" /* U2V2 U2V2 U3V3 U3V3*/
2656 "movq 8(%1, %0, 4), %%mm0\n\t"
2657 "movq %%mm0, %%mm3\n\t"
2658 "punpcklbw %%mm6, %%mm0\n\t" /* Y U2 Y V2 Y U2 Y V2*/
2659 "punpckhbw %%mm6, %%mm3\n\t" /* Y U3 Y V3 Y U3 Y V3*/
2660 MOVNTQ" %%mm0, 16(%4, %0, 8)\n\t"
2661 MOVNTQ" %%mm3, 24(%4, %0, 8)\n\t"
2663 "movq %%mm4, %%mm6\n\t"
2664 "movq 16(%1, %0, 4), %%mm0\n\t"
2665 "movq %%mm0, %%mm3\n\t"
2666 "punpcklbw %%mm5, %%mm4\n\t"
2667 "punpcklbw %%mm4, %%mm0\n\t" /* Y U4 Y V4 Y U4 Y V4*/
2668 "punpckhbw %%mm4, %%mm3\n\t" /* Y U5 Y V5 Y U5 Y V5*/
2669 MOVNTQ" %%mm0, 32(%4, %0, 8)\n\t"
2670 MOVNTQ" %%mm3, 40(%4, %0, 8)\n\t"
2672 "punpckhbw %%mm5, %%mm6\n\t"
2673 "movq 24(%1, %0, 4), %%mm0\n\t"
2674 "movq %%mm0, %%mm3\n\t"
2675 "punpcklbw %%mm6, %%mm0\n\t" /* Y U6 Y V6 Y U6 Y V6*/
2676 "punpckhbw %%mm6, %%mm3\n\t" /* Y U7 Y V7 Y U7 Y V7*/
2677 MOVNTQ" %%mm0, 48(%4, %0, 8)\n\t"
2678 MOVNTQ" %%mm3, 56(%4, %0, 8)\n\t"
2681 : "r"(yp), "r" (up), "r"(vp), "r"(d)
2687 const long x2= x<<2;
2707 static inline void RENAME(rgb2rgb_init)(void){
2708 rgb15to16= RENAME(rgb15to16);
2709 rgb15to24= RENAME(rgb15to24);
2710 rgb15to32= RENAME(rgb15to32);
2711 rgb16to24= RENAME(rgb16to24);
2712 rgb16to32= RENAME(rgb16to32);
2713 rgb16to15= RENAME(rgb16to15);
2714 rgb24to16= RENAME(rgb24to16);
2715 rgb24to15= RENAME(rgb24to15);
2716 rgb24to32= RENAME(rgb24to32);
2717 rgb32to16= RENAME(rgb32to16);
2718 rgb32to15= RENAME(rgb32to15);
2719 rgb32to24= RENAME(rgb32to24);
2720 rgb24tobgr15= RENAME(rgb24tobgr15);
2721 rgb24tobgr16= RENAME(rgb24tobgr16);
2722 rgb24tobgr24= RENAME(rgb24tobgr24);
2723 rgb32tobgr32= RENAME(rgb32tobgr32);
2724 rgb32tobgr16= RENAME(rgb32tobgr16);
2725 rgb32tobgr15= RENAME(rgb32tobgr15);
2726 yv12toyuy2= RENAME(yv12toyuy2);
2727 yv12touyvy= RENAME(yv12touyvy);
2728 yuv422ptoyuy2= RENAME(yuv422ptoyuy2);
2729 yuy2toyv12= RENAME(yuy2toyv12);
2730 // uyvytoyv12= RENAME(uyvytoyv12);
2731 // yvu9toyv12= RENAME(yvu9toyv12);
2732 planar2x= RENAME(planar2x);
2733 rgb24toyv12= RENAME(rgb24toyv12);
2734 interleaveBytes= RENAME(interleaveBytes);
2735 vu9_to_vu12= RENAME(vu9_to_vu12);
2736 yvu9_to_yuy2= RENAME(yvu9_to_yuy2);