3 * rgb2rgb.c, Software RGB to RGB convertor
4 * pluralize by Software PAL8 to RGB convertor
5 * Software YUV to YUV convertor
6 * Software YUV to RGB convertor
7 * Written by Nick Kurshev.
8 * palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
9 * lot of big-endian byteorder fixes by Alex Beregszaszi
11 * This file is part of FFmpeg.
13 * FFmpeg is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
18 * FFmpeg is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * You should have received a copy of the GNU General Public License
24 * along with FFmpeg; if not, write to the Free Software
25 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
29 #include <inttypes.h> /* for __WORDSIZE */
32 // #warning You have misconfigured system and probably will lose performance!
33 #define __WORDSIZE MP_WORDSIZE
51 #define PREFETCH "prefetch"
52 #define PREFETCHW "prefetchw"
53 #define PAVGB "pavgusb"
54 #elif defined ( HAVE_MMX2 )
55 #define PREFETCH "prefetchnta"
56 #define PREFETCHW "prefetcht0"
63 #define PREFETCH "/nop"
64 #define PREFETCHW "/nop"
69 /* On K6 femms is faster of emms. On K7 femms is directly mapped on emms. */
76 #define MOVNTQ "movntq"
77 #define SFENCE "sfence"
87 static inline void RENAME(rgb24to32)(const uint8_t *src,uint8_t *dst,long src_size)
90 const uint8_t *s = src;
93 const uint8_t *mm_end;
97 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
99 __asm __volatile("movq %0, %%mm7"::"m"(mask32):"memory");
105 "punpckldq 3%1, %%mm0\n\t"
106 "movd 6%1, %%mm1\n\t"
107 "punpckldq 9%1, %%mm1\n\t"
108 "movd 12%1, %%mm2\n\t"
109 "punpckldq 15%1, %%mm2\n\t"
110 "movd 18%1, %%mm3\n\t"
111 "punpckldq 21%1, %%mm3\n\t"
112 "pand %%mm7, %%mm0\n\t"
113 "pand %%mm7, %%mm1\n\t"
114 "pand %%mm7, %%mm2\n\t"
115 "pand %%mm7, %%mm3\n\t"
116 MOVNTQ" %%mm0, %0\n\t"
117 MOVNTQ" %%mm1, 8%0\n\t"
118 MOVNTQ" %%mm2, 16%0\n\t"
126 __asm __volatile(SFENCE:::"memory");
127 __asm __volatile(EMMS:::"memory");
131 #ifdef WORDS_BIGENDIAN
132 /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */
147 static inline void RENAME(rgb32to24)(const uint8_t *src,uint8_t *dst,long src_size)
150 const uint8_t *s = src;
153 const uint8_t *mm_end;
157 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
164 "movq 8%1, %%mm1\n\t"
165 "movq 16%1, %%mm4\n\t"
166 "movq 24%1, %%mm5\n\t"
167 "movq %%mm0, %%mm2\n\t"
168 "movq %%mm1, %%mm3\n\t"
169 "movq %%mm4, %%mm6\n\t"
170 "movq %%mm5, %%mm7\n\t"
171 "psrlq $8, %%mm2\n\t"
172 "psrlq $8, %%mm3\n\t"
173 "psrlq $8, %%mm6\n\t"
174 "psrlq $8, %%mm7\n\t"
183 "por %%mm2, %%mm0\n\t"
184 "por %%mm3, %%mm1\n\t"
185 "por %%mm6, %%mm4\n\t"
186 "por %%mm7, %%mm5\n\t"
188 "movq %%mm1, %%mm2\n\t"
189 "movq %%mm4, %%mm3\n\t"
190 "psllq $48, %%mm2\n\t"
191 "psllq $32, %%mm3\n\t"
194 "por %%mm2, %%mm0\n\t"
195 "psrlq $16, %%mm1\n\t"
196 "psrlq $32, %%mm4\n\t"
197 "psllq $16, %%mm5\n\t"
198 "por %%mm3, %%mm1\n\t"
200 "por %%mm5, %%mm4\n\t"
202 MOVNTQ" %%mm0, %0\n\t"
203 MOVNTQ" %%mm1, 8%0\n\t"
206 :"m"(*s),"m"(mask24l),
207 "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
212 __asm __volatile(SFENCE:::"memory");
213 __asm __volatile(EMMS:::"memory");
217 #ifdef WORDS_BIGENDIAN
218 /* RGB32 (= A,B,G,R) -> RGB24 (= R,G,B) */
234 Original by Strepto/Astral
235 ported to gcc & bugfixed : A'rpi
236 MMX2, 3DNOW optimization by Nick Kurshev
237 32bit c version, and and&add trick by Michael Niedermayer
239 static inline void RENAME(rgb15to16)(const uint8_t *src,uint8_t *dst,long src_size)
241 register const uint8_t* s=src;
242 register uint8_t* d=dst;
243 register const uint8_t *end;
244 const uint8_t *mm_end;
247 __asm __volatile(PREFETCH" %0"::"m"(*s));
248 __asm __volatile("movq %0, %%mm4"::"m"(mask15s));
255 "movq 8%1, %%mm2\n\t"
256 "movq %%mm0, %%mm1\n\t"
257 "movq %%mm2, %%mm3\n\t"
258 "pand %%mm4, %%mm0\n\t"
259 "pand %%mm4, %%mm2\n\t"
260 "paddw %%mm1, %%mm0\n\t"
261 "paddw %%mm3, %%mm2\n\t"
262 MOVNTQ" %%mm0, %0\n\t"
270 __asm __volatile(SFENCE:::"memory");
271 __asm __volatile(EMMS:::"memory");
276 register unsigned x= *((uint32_t *)s);
277 *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
283 register unsigned short x= *((uint16_t *)s);
284 *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
288 static inline void RENAME(rgb16to15)(const uint8_t *src,uint8_t *dst,long src_size)
290 register const uint8_t* s=src;
291 register uint8_t* d=dst;
292 register const uint8_t *end;
293 const uint8_t *mm_end;
296 __asm __volatile(PREFETCH" %0"::"m"(*s));
297 __asm __volatile("movq %0, %%mm7"::"m"(mask15rg));
298 __asm __volatile("movq %0, %%mm6"::"m"(mask15b));
305 "movq 8%1, %%mm2\n\t"
306 "movq %%mm0, %%mm1\n\t"
307 "movq %%mm2, %%mm3\n\t"
308 "psrlq $1, %%mm0\n\t"
309 "psrlq $1, %%mm2\n\t"
310 "pand %%mm7, %%mm0\n\t"
311 "pand %%mm7, %%mm2\n\t"
312 "pand %%mm6, %%mm1\n\t"
313 "pand %%mm6, %%mm3\n\t"
314 "por %%mm1, %%mm0\n\t"
315 "por %%mm3, %%mm2\n\t"
316 MOVNTQ" %%mm0, %0\n\t"
324 __asm __volatile(SFENCE:::"memory");
325 __asm __volatile(EMMS:::"memory");
330 register uint32_t x= *((uint32_t *)s);
331 *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
337 register uint16_t x= *((uint16_t *)s);
338 *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
344 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, long src_size)
346 const uint8_t *s = src;
349 const uint8_t *mm_end;
351 uint16_t *d = (uint16_t *)dst;
355 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
357 "movq %3, %%mm5 \n\t"
358 "movq %4, %%mm6 \n\t"
359 "movq %5, %%mm7 \n\t"
362 PREFETCH" 32(%1) \n\t"
363 "movd (%1), %%mm0 \n\t"
364 "movd 4(%1), %%mm3 \n\t"
365 "punpckldq 8(%1), %%mm0 \n\t"
366 "punpckldq 12(%1), %%mm3 \n\t"
367 "movq %%mm0, %%mm1 \n\t"
368 "movq %%mm3, %%mm4 \n\t"
369 "pand %%mm6, %%mm0 \n\t"
370 "pand %%mm6, %%mm3 \n\t"
371 "pmaddwd %%mm7, %%mm0 \n\t"
372 "pmaddwd %%mm7, %%mm3 \n\t"
373 "pand %%mm5, %%mm1 \n\t"
374 "pand %%mm5, %%mm4 \n\t"
375 "por %%mm1, %%mm0 \n\t"
376 "por %%mm4, %%mm3 \n\t"
377 "psrld $5, %%mm0 \n\t"
378 "pslld $11, %%mm3 \n\t"
379 "por %%mm3, %%mm0 \n\t"
380 MOVNTQ" %%mm0, (%0) \n\t"
386 : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
389 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
393 ::"m"(red_16mask),"m"(green_16mask));
399 "movd 4%1, %%mm3\n\t"
400 "punpckldq 8%1, %%mm0\n\t"
401 "punpckldq 12%1, %%mm3\n\t"
402 "movq %%mm0, %%mm1\n\t"
403 "movq %%mm0, %%mm2\n\t"
404 "movq %%mm3, %%mm4\n\t"
405 "movq %%mm3, %%mm5\n\t"
406 "psrlq $3, %%mm0\n\t"
407 "psrlq $3, %%mm3\n\t"
410 "psrlq $5, %%mm1\n\t"
411 "psrlq $5, %%mm4\n\t"
412 "pand %%mm6, %%mm1\n\t"
413 "pand %%mm6, %%mm4\n\t"
414 "psrlq $8, %%mm2\n\t"
415 "psrlq $8, %%mm5\n\t"
416 "pand %%mm7, %%mm2\n\t"
417 "pand %%mm7, %%mm5\n\t"
418 "por %%mm1, %%mm0\n\t"
419 "por %%mm4, %%mm3\n\t"
420 "por %%mm2, %%mm0\n\t"
421 "por %%mm5, %%mm3\n\t"
422 "psllq $16, %%mm3\n\t"
423 "por %%mm3, %%mm0\n\t"
424 MOVNTQ" %%mm0, %0\n\t"
425 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
430 __asm __volatile(SFENCE:::"memory");
431 __asm __volatile(EMMS:::"memory");
435 register int rgb = *(uint32_t*)s; s += 4;
436 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
440 static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
442 const uint8_t *s = src;
445 const uint8_t *mm_end;
447 uint16_t *d = (uint16_t *)dst;
450 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
454 ::"m"(red_16mask),"m"(green_16mask));
461 "movd 4%1, %%mm3\n\t"
462 "punpckldq 8%1, %%mm0\n\t"
463 "punpckldq 12%1, %%mm3\n\t"
464 "movq %%mm0, %%mm1\n\t"
465 "movq %%mm0, %%mm2\n\t"
466 "movq %%mm3, %%mm4\n\t"
467 "movq %%mm3, %%mm5\n\t"
468 "psllq $8, %%mm0\n\t"
469 "psllq $8, %%mm3\n\t"
470 "pand %%mm7, %%mm0\n\t"
471 "pand %%mm7, %%mm3\n\t"
472 "psrlq $5, %%mm1\n\t"
473 "psrlq $5, %%mm4\n\t"
474 "pand %%mm6, %%mm1\n\t"
475 "pand %%mm6, %%mm4\n\t"
476 "psrlq $19, %%mm2\n\t"
477 "psrlq $19, %%mm5\n\t"
480 "por %%mm1, %%mm0\n\t"
481 "por %%mm4, %%mm3\n\t"
482 "por %%mm2, %%mm0\n\t"
483 "por %%mm5, %%mm3\n\t"
484 "psllq $16, %%mm3\n\t"
485 "por %%mm3, %%mm0\n\t"
486 MOVNTQ" %%mm0, %0\n\t"
487 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
491 __asm __volatile(SFENCE:::"memory");
492 __asm __volatile(EMMS:::"memory");
496 register int rgb = *(uint32_t*)s; s += 4;
497 *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
501 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, long src_size)
503 const uint8_t *s = src;
506 const uint8_t *mm_end;
508 uint16_t *d = (uint16_t *)dst;
512 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
514 "movq %3, %%mm5 \n\t"
515 "movq %4, %%mm6 \n\t"
516 "movq %5, %%mm7 \n\t"
519 PREFETCH" 32(%1) \n\t"
520 "movd (%1), %%mm0 \n\t"
521 "movd 4(%1), %%mm3 \n\t"
522 "punpckldq 8(%1), %%mm0 \n\t"
523 "punpckldq 12(%1), %%mm3 \n\t"
524 "movq %%mm0, %%mm1 \n\t"
525 "movq %%mm3, %%mm4 \n\t"
526 "pand %%mm6, %%mm0 \n\t"
527 "pand %%mm6, %%mm3 \n\t"
528 "pmaddwd %%mm7, %%mm0 \n\t"
529 "pmaddwd %%mm7, %%mm3 \n\t"
530 "pand %%mm5, %%mm1 \n\t"
531 "pand %%mm5, %%mm4 \n\t"
532 "por %%mm1, %%mm0 \n\t"
533 "por %%mm4, %%mm3 \n\t"
534 "psrld $6, %%mm0 \n\t"
535 "pslld $10, %%mm3 \n\t"
536 "por %%mm3, %%mm0 \n\t"
537 MOVNTQ" %%mm0, (%0) \n\t"
543 : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
546 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
550 ::"m"(red_15mask),"m"(green_15mask));
556 "movd 4%1, %%mm3\n\t"
557 "punpckldq 8%1, %%mm0\n\t"
558 "punpckldq 12%1, %%mm3\n\t"
559 "movq %%mm0, %%mm1\n\t"
560 "movq %%mm0, %%mm2\n\t"
561 "movq %%mm3, %%mm4\n\t"
562 "movq %%mm3, %%mm5\n\t"
563 "psrlq $3, %%mm0\n\t"
564 "psrlq $3, %%mm3\n\t"
567 "psrlq $6, %%mm1\n\t"
568 "psrlq $6, %%mm4\n\t"
569 "pand %%mm6, %%mm1\n\t"
570 "pand %%mm6, %%mm4\n\t"
571 "psrlq $9, %%mm2\n\t"
572 "psrlq $9, %%mm5\n\t"
573 "pand %%mm7, %%mm2\n\t"
574 "pand %%mm7, %%mm5\n\t"
575 "por %%mm1, %%mm0\n\t"
576 "por %%mm4, %%mm3\n\t"
577 "por %%mm2, %%mm0\n\t"
578 "por %%mm5, %%mm3\n\t"
579 "psllq $16, %%mm3\n\t"
580 "por %%mm3, %%mm0\n\t"
581 MOVNTQ" %%mm0, %0\n\t"
582 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
587 __asm __volatile(SFENCE:::"memory");
588 __asm __volatile(EMMS:::"memory");
592 register int rgb = *(uint32_t*)s; s += 4;
593 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
597 static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
599 const uint8_t *s = src;
602 const uint8_t *mm_end;
604 uint16_t *d = (uint16_t *)dst;
607 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
611 ::"m"(red_15mask),"m"(green_15mask));
618 "movd 4%1, %%mm3\n\t"
619 "punpckldq 8%1, %%mm0\n\t"
620 "punpckldq 12%1, %%mm3\n\t"
621 "movq %%mm0, %%mm1\n\t"
622 "movq %%mm0, %%mm2\n\t"
623 "movq %%mm3, %%mm4\n\t"
624 "movq %%mm3, %%mm5\n\t"
625 "psllq $7, %%mm0\n\t"
626 "psllq $7, %%mm3\n\t"
627 "pand %%mm7, %%mm0\n\t"
628 "pand %%mm7, %%mm3\n\t"
629 "psrlq $6, %%mm1\n\t"
630 "psrlq $6, %%mm4\n\t"
631 "pand %%mm6, %%mm1\n\t"
632 "pand %%mm6, %%mm4\n\t"
633 "psrlq $19, %%mm2\n\t"
634 "psrlq $19, %%mm5\n\t"
637 "por %%mm1, %%mm0\n\t"
638 "por %%mm4, %%mm3\n\t"
639 "por %%mm2, %%mm0\n\t"
640 "por %%mm5, %%mm3\n\t"
641 "psllq $16, %%mm3\n\t"
642 "por %%mm3, %%mm0\n\t"
643 MOVNTQ" %%mm0, %0\n\t"
644 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
648 __asm __volatile(SFENCE:::"memory");
649 __asm __volatile(EMMS:::"memory");
653 register int rgb = *(uint32_t*)s; s += 4;
654 *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
658 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, long src_size)
660 const uint8_t *s = src;
663 const uint8_t *mm_end;
665 uint16_t *d = (uint16_t *)dst;
668 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
672 ::"m"(red_16mask),"m"(green_16mask));
679 "movd 3%1, %%mm3\n\t"
680 "punpckldq 6%1, %%mm0\n\t"
681 "punpckldq 9%1, %%mm3\n\t"
682 "movq %%mm0, %%mm1\n\t"
683 "movq %%mm0, %%mm2\n\t"
684 "movq %%mm3, %%mm4\n\t"
685 "movq %%mm3, %%mm5\n\t"
686 "psrlq $3, %%mm0\n\t"
687 "psrlq $3, %%mm3\n\t"
690 "psrlq $5, %%mm1\n\t"
691 "psrlq $5, %%mm4\n\t"
692 "pand %%mm6, %%mm1\n\t"
693 "pand %%mm6, %%mm4\n\t"
694 "psrlq $8, %%mm2\n\t"
695 "psrlq $8, %%mm5\n\t"
696 "pand %%mm7, %%mm2\n\t"
697 "pand %%mm7, %%mm5\n\t"
698 "por %%mm1, %%mm0\n\t"
699 "por %%mm4, %%mm3\n\t"
700 "por %%mm2, %%mm0\n\t"
701 "por %%mm5, %%mm3\n\t"
702 "psllq $16, %%mm3\n\t"
703 "por %%mm3, %%mm0\n\t"
704 MOVNTQ" %%mm0, %0\n\t"
705 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
709 __asm __volatile(SFENCE:::"memory");
710 __asm __volatile(EMMS:::"memory");
717 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
721 static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
723 const uint8_t *s = src;
726 const uint8_t *mm_end;
728 uint16_t *d = (uint16_t *)dst;
731 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
735 ::"m"(red_16mask),"m"(green_16mask));
742 "movd 3%1, %%mm3\n\t"
743 "punpckldq 6%1, %%mm0\n\t"
744 "punpckldq 9%1, %%mm3\n\t"
745 "movq %%mm0, %%mm1\n\t"
746 "movq %%mm0, %%mm2\n\t"
747 "movq %%mm3, %%mm4\n\t"
748 "movq %%mm3, %%mm5\n\t"
749 "psllq $8, %%mm0\n\t"
750 "psllq $8, %%mm3\n\t"
751 "pand %%mm7, %%mm0\n\t"
752 "pand %%mm7, %%mm3\n\t"
753 "psrlq $5, %%mm1\n\t"
754 "psrlq $5, %%mm4\n\t"
755 "pand %%mm6, %%mm1\n\t"
756 "pand %%mm6, %%mm4\n\t"
757 "psrlq $19, %%mm2\n\t"
758 "psrlq $19, %%mm5\n\t"
761 "por %%mm1, %%mm0\n\t"
762 "por %%mm4, %%mm3\n\t"
763 "por %%mm2, %%mm0\n\t"
764 "por %%mm5, %%mm3\n\t"
765 "psllq $16, %%mm3\n\t"
766 "por %%mm3, %%mm0\n\t"
767 MOVNTQ" %%mm0, %0\n\t"
768 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
772 __asm __volatile(SFENCE:::"memory");
773 __asm __volatile(EMMS:::"memory");
780 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
784 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, long src_size)
786 const uint8_t *s = src;
789 const uint8_t *mm_end;
791 uint16_t *d = (uint16_t *)dst;
794 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
798 ::"m"(red_15mask),"m"(green_15mask));
805 "movd 3%1, %%mm3\n\t"
806 "punpckldq 6%1, %%mm0\n\t"
807 "punpckldq 9%1, %%mm3\n\t"
808 "movq %%mm0, %%mm1\n\t"
809 "movq %%mm0, %%mm2\n\t"
810 "movq %%mm3, %%mm4\n\t"
811 "movq %%mm3, %%mm5\n\t"
812 "psrlq $3, %%mm0\n\t"
813 "psrlq $3, %%mm3\n\t"
816 "psrlq $6, %%mm1\n\t"
817 "psrlq $6, %%mm4\n\t"
818 "pand %%mm6, %%mm1\n\t"
819 "pand %%mm6, %%mm4\n\t"
820 "psrlq $9, %%mm2\n\t"
821 "psrlq $9, %%mm5\n\t"
822 "pand %%mm7, %%mm2\n\t"
823 "pand %%mm7, %%mm5\n\t"
824 "por %%mm1, %%mm0\n\t"
825 "por %%mm4, %%mm3\n\t"
826 "por %%mm2, %%mm0\n\t"
827 "por %%mm5, %%mm3\n\t"
828 "psllq $16, %%mm3\n\t"
829 "por %%mm3, %%mm0\n\t"
830 MOVNTQ" %%mm0, %0\n\t"
831 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
835 __asm __volatile(SFENCE:::"memory");
836 __asm __volatile(EMMS:::"memory");
843 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
847 static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
849 const uint8_t *s = src;
852 const uint8_t *mm_end;
854 uint16_t *d = (uint16_t *)dst;
857 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
861 ::"m"(red_15mask),"m"(green_15mask));
868 "movd 3%1, %%mm3\n\t"
869 "punpckldq 6%1, %%mm0\n\t"
870 "punpckldq 9%1, %%mm3\n\t"
871 "movq %%mm0, %%mm1\n\t"
872 "movq %%mm0, %%mm2\n\t"
873 "movq %%mm3, %%mm4\n\t"
874 "movq %%mm3, %%mm5\n\t"
875 "psllq $7, %%mm0\n\t"
876 "psllq $7, %%mm3\n\t"
877 "pand %%mm7, %%mm0\n\t"
878 "pand %%mm7, %%mm3\n\t"
879 "psrlq $6, %%mm1\n\t"
880 "psrlq $6, %%mm4\n\t"
881 "pand %%mm6, %%mm1\n\t"
882 "pand %%mm6, %%mm4\n\t"
883 "psrlq $19, %%mm2\n\t"
884 "psrlq $19, %%mm5\n\t"
887 "por %%mm1, %%mm0\n\t"
888 "por %%mm4, %%mm3\n\t"
889 "por %%mm2, %%mm0\n\t"
890 "por %%mm5, %%mm3\n\t"
891 "psllq $16, %%mm3\n\t"
892 "por %%mm3, %%mm0\n\t"
893 MOVNTQ" %%mm0, %0\n\t"
894 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
898 __asm __volatile(SFENCE:::"memory");
899 __asm __volatile(EMMS:::"memory");
906 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
911 I use here less accurate approximation by simply
912 left-shifting the input
913 value and filling the low order bits with
914 zeroes. This method improves png's
915 compression but this scheme cannot reproduce white exactly, since it does not
916 generate an all-ones maximum value; the net effect is to darken the
919 The better method should be "left bit replication":
929 | Leftmost Bits Repeated to Fill Open Bits
933 static inline void RENAME(rgb15to24)(const uint8_t *src, uint8_t *dst, long src_size)
937 const uint16_t *mm_end;
939 uint8_t *d = (uint8_t *)dst;
940 const uint16_t *s = (uint16_t *)src;
941 end = s + src_size/2;
943 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
955 "psllq $3, %%mm0\n\t"
956 "psrlq $2, %%mm1\n\t"
957 "psrlq $7, %%mm2\n\t"
958 "movq %%mm0, %%mm3\n\t"
959 "movq %%mm1, %%mm4\n\t"
960 "movq %%mm2, %%mm5\n\t"
961 "punpcklwd %5, %%mm0\n\t"
962 "punpcklwd %5, %%mm1\n\t"
963 "punpcklwd %5, %%mm2\n\t"
964 "punpckhwd %5, %%mm3\n\t"
965 "punpckhwd %5, %%mm4\n\t"
966 "punpckhwd %5, %%mm5\n\t"
967 "psllq $8, %%mm1\n\t"
968 "psllq $16, %%mm2\n\t"
969 "por %%mm1, %%mm0\n\t"
970 "por %%mm2, %%mm0\n\t"
971 "psllq $8, %%mm4\n\t"
972 "psllq $16, %%mm5\n\t"
973 "por %%mm4, %%mm3\n\t"
974 "por %%mm5, %%mm3\n\t"
976 "movq %%mm0, %%mm6\n\t"
977 "movq %%mm3, %%mm7\n\t"
979 "movq 8%1, %%mm0\n\t"
980 "movq 8%1, %%mm1\n\t"
981 "movq 8%1, %%mm2\n\t"
985 "psllq $3, %%mm0\n\t"
986 "psrlq $2, %%mm1\n\t"
987 "psrlq $7, %%mm2\n\t"
988 "movq %%mm0, %%mm3\n\t"
989 "movq %%mm1, %%mm4\n\t"
990 "movq %%mm2, %%mm5\n\t"
991 "punpcklwd %5, %%mm0\n\t"
992 "punpcklwd %5, %%mm1\n\t"
993 "punpcklwd %5, %%mm2\n\t"
994 "punpckhwd %5, %%mm3\n\t"
995 "punpckhwd %5, %%mm4\n\t"
996 "punpckhwd %5, %%mm5\n\t"
997 "psllq $8, %%mm1\n\t"
998 "psllq $16, %%mm2\n\t"
999 "por %%mm1, %%mm0\n\t"
1000 "por %%mm2, %%mm0\n\t"
1001 "psllq $8, %%mm4\n\t"
1002 "psllq $16, %%mm5\n\t"
1003 "por %%mm4, %%mm3\n\t"
1004 "por %%mm5, %%mm3\n\t"
1007 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)
1009 /* Borrowed 32 to 24 */
1011 "movq %%mm0, %%mm4\n\t"
1012 "movq %%mm3, %%mm5\n\t"
1013 "movq %%mm6, %%mm0\n\t"
1014 "movq %%mm7, %%mm1\n\t"
1016 "movq %%mm4, %%mm6\n\t"
1017 "movq %%mm5, %%mm7\n\t"
1018 "movq %%mm0, %%mm2\n\t"
1019 "movq %%mm1, %%mm3\n\t"
1021 "psrlq $8, %%mm2\n\t"
1022 "psrlq $8, %%mm3\n\t"
1023 "psrlq $8, %%mm6\n\t"
1024 "psrlq $8, %%mm7\n\t"
1025 "pand %2, %%mm0\n\t"
1026 "pand %2, %%mm1\n\t"
1027 "pand %2, %%mm4\n\t"
1028 "pand %2, %%mm5\n\t"
1029 "pand %3, %%mm2\n\t"
1030 "pand %3, %%mm3\n\t"
1031 "pand %3, %%mm6\n\t"
1032 "pand %3, %%mm7\n\t"
1033 "por %%mm2, %%mm0\n\t"
1034 "por %%mm3, %%mm1\n\t"
1035 "por %%mm6, %%mm4\n\t"
1036 "por %%mm7, %%mm5\n\t"
1038 "movq %%mm1, %%mm2\n\t"
1039 "movq %%mm4, %%mm3\n\t"
1040 "psllq $48, %%mm2\n\t"
1041 "psllq $32, %%mm3\n\t"
1042 "pand %4, %%mm2\n\t"
1043 "pand %5, %%mm3\n\t"
1044 "por %%mm2, %%mm0\n\t"
1045 "psrlq $16, %%mm1\n\t"
1046 "psrlq $32, %%mm4\n\t"
1047 "psllq $16, %%mm5\n\t"
1048 "por %%mm3, %%mm1\n\t"
1049 "pand %6, %%mm5\n\t"
1050 "por %%mm5, %%mm4\n\t"
1052 MOVNTQ" %%mm0, %0\n\t"
1053 MOVNTQ" %%mm1, 8%0\n\t"
1054 MOVNTQ" %%mm4, 16%0"
1057 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1062 __asm __volatile(SFENCE:::"memory");
1063 __asm __volatile(EMMS:::"memory");
1067 register uint16_t bgr;
1069 *d++ = (bgr&0x1F)<<3;
1070 *d++ = (bgr&0x3E0)>>2;
1071 *d++ = (bgr&0x7C00)>>7;
1075 static inline void RENAME(rgb16to24)(const uint8_t *src, uint8_t *dst, long src_size)
1077 const uint16_t *end;
1079 const uint16_t *mm_end;
1081 uint8_t *d = (uint8_t *)dst;
1082 const uint16_t *s = (const uint16_t *)src;
1083 end = s + src_size/2;
1085 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
1091 "movq %1, %%mm0\n\t"
1092 "movq %1, %%mm1\n\t"
1093 "movq %1, %%mm2\n\t"
1094 "pand %2, %%mm0\n\t"
1095 "pand %3, %%mm1\n\t"
1096 "pand %4, %%mm2\n\t"
1097 "psllq $3, %%mm0\n\t"
1098 "psrlq $3, %%mm1\n\t"
1099 "psrlq $8, %%mm2\n\t"
1100 "movq %%mm0, %%mm3\n\t"
1101 "movq %%mm1, %%mm4\n\t"
1102 "movq %%mm2, %%mm5\n\t"
1103 "punpcklwd %5, %%mm0\n\t"
1104 "punpcklwd %5, %%mm1\n\t"
1105 "punpcklwd %5, %%mm2\n\t"
1106 "punpckhwd %5, %%mm3\n\t"
1107 "punpckhwd %5, %%mm4\n\t"
1108 "punpckhwd %5, %%mm5\n\t"
1109 "psllq $8, %%mm1\n\t"
1110 "psllq $16, %%mm2\n\t"
1111 "por %%mm1, %%mm0\n\t"
1112 "por %%mm2, %%mm0\n\t"
1113 "psllq $8, %%mm4\n\t"
1114 "psllq $16, %%mm5\n\t"
1115 "por %%mm4, %%mm3\n\t"
1116 "por %%mm5, %%mm3\n\t"
1118 "movq %%mm0, %%mm6\n\t"
1119 "movq %%mm3, %%mm7\n\t"
1121 "movq 8%1, %%mm0\n\t"
1122 "movq 8%1, %%mm1\n\t"
1123 "movq 8%1, %%mm2\n\t"
1124 "pand %2, %%mm0\n\t"
1125 "pand %3, %%mm1\n\t"
1126 "pand %4, %%mm2\n\t"
1127 "psllq $3, %%mm0\n\t"
1128 "psrlq $3, %%mm1\n\t"
1129 "psrlq $8, %%mm2\n\t"
1130 "movq %%mm0, %%mm3\n\t"
1131 "movq %%mm1, %%mm4\n\t"
1132 "movq %%mm2, %%mm5\n\t"
1133 "punpcklwd %5, %%mm0\n\t"
1134 "punpcklwd %5, %%mm1\n\t"
1135 "punpcklwd %5, %%mm2\n\t"
1136 "punpckhwd %5, %%mm3\n\t"
1137 "punpckhwd %5, %%mm4\n\t"
1138 "punpckhwd %5, %%mm5\n\t"
1139 "psllq $8, %%mm1\n\t"
1140 "psllq $16, %%mm2\n\t"
1141 "por %%mm1, %%mm0\n\t"
1142 "por %%mm2, %%mm0\n\t"
1143 "psllq $8, %%mm4\n\t"
1144 "psllq $16, %%mm5\n\t"
1145 "por %%mm4, %%mm3\n\t"
1146 "por %%mm5, %%mm3\n\t"
1148 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)
1150 /* Borrowed 32 to 24 */
1152 "movq %%mm0, %%mm4\n\t"
1153 "movq %%mm3, %%mm5\n\t"
1154 "movq %%mm6, %%mm0\n\t"
1155 "movq %%mm7, %%mm1\n\t"
1157 "movq %%mm4, %%mm6\n\t"
1158 "movq %%mm5, %%mm7\n\t"
1159 "movq %%mm0, %%mm2\n\t"
1160 "movq %%mm1, %%mm3\n\t"
1162 "psrlq $8, %%mm2\n\t"
1163 "psrlq $8, %%mm3\n\t"
1164 "psrlq $8, %%mm6\n\t"
1165 "psrlq $8, %%mm7\n\t"
1166 "pand %2, %%mm0\n\t"
1167 "pand %2, %%mm1\n\t"
1168 "pand %2, %%mm4\n\t"
1169 "pand %2, %%mm5\n\t"
1170 "pand %3, %%mm2\n\t"
1171 "pand %3, %%mm3\n\t"
1172 "pand %3, %%mm6\n\t"
1173 "pand %3, %%mm7\n\t"
1174 "por %%mm2, %%mm0\n\t"
1175 "por %%mm3, %%mm1\n\t"
1176 "por %%mm6, %%mm4\n\t"
1177 "por %%mm7, %%mm5\n\t"
1179 "movq %%mm1, %%mm2\n\t"
1180 "movq %%mm4, %%mm3\n\t"
1181 "psllq $48, %%mm2\n\t"
1182 "psllq $32, %%mm3\n\t"
1183 "pand %4, %%mm2\n\t"
1184 "pand %5, %%mm3\n\t"
1185 "por %%mm2, %%mm0\n\t"
1186 "psrlq $16, %%mm1\n\t"
1187 "psrlq $32, %%mm4\n\t"
1188 "psllq $16, %%mm5\n\t"
1189 "por %%mm3, %%mm1\n\t"
1190 "pand %6, %%mm5\n\t"
1191 "por %%mm5, %%mm4\n\t"
1193 MOVNTQ" %%mm0, %0\n\t"
1194 MOVNTQ" %%mm1, 8%0\n\t"
1195 MOVNTQ" %%mm4, 16%0"
1198 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1203 __asm __volatile(SFENCE:::"memory");
1204 __asm __volatile(EMMS:::"memory");
1208 register uint16_t bgr;
1210 *d++ = (bgr&0x1F)<<3;
1211 *d++ = (bgr&0x7E0)>>3;
1212 *d++ = (bgr&0xF800)>>8;
1216 static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_size)
1218 const uint16_t *end;
1220 const uint16_t *mm_end;
1222 uint8_t *d = (uint8_t *)dst;
1223 const uint16_t *s = (const uint16_t *)src;
1224 end = s + src_size/2;
1226 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
1227 __asm __volatile("pxor %%mm7,%%mm7\n\t":::"memory");
1233 "movq %1, %%mm0\n\t"
1234 "movq %1, %%mm1\n\t"
1235 "movq %1, %%mm2\n\t"
1236 "pand %2, %%mm0\n\t"
1237 "pand %3, %%mm1\n\t"
1238 "pand %4, %%mm2\n\t"
1239 "psllq $3, %%mm0\n\t"
1240 "psrlq $2, %%mm1\n\t"
1241 "psrlq $7, %%mm2\n\t"
1242 "movq %%mm0, %%mm3\n\t"
1243 "movq %%mm1, %%mm4\n\t"
1244 "movq %%mm2, %%mm5\n\t"
1245 "punpcklwd %%mm7, %%mm0\n\t"
1246 "punpcklwd %%mm7, %%mm1\n\t"
1247 "punpcklwd %%mm7, %%mm2\n\t"
1248 "punpckhwd %%mm7, %%mm3\n\t"
1249 "punpckhwd %%mm7, %%mm4\n\t"
1250 "punpckhwd %%mm7, %%mm5\n\t"
1251 "psllq $8, %%mm1\n\t"
1252 "psllq $16, %%mm2\n\t"
1253 "por %%mm1, %%mm0\n\t"
1254 "por %%mm2, %%mm0\n\t"
1255 "psllq $8, %%mm4\n\t"
1256 "psllq $16, %%mm5\n\t"
1257 "por %%mm4, %%mm3\n\t"
1258 "por %%mm5, %%mm3\n\t"
1259 MOVNTQ" %%mm0, %0\n\t"
1260 MOVNTQ" %%mm3, 8%0\n\t"
1262 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r)
1267 __asm __volatile(SFENCE:::"memory");
1268 __asm __volatile(EMMS:::"memory");
1272 #if 0 //slightly slower on athlon
1274 *((uint32_t*)d)++ = ((bgr&0x1F)<<3) + ((bgr&0x3E0)<<6) + ((bgr&0x7C00)<<9);
1276 register uint16_t bgr;
1278 #ifdef WORDS_BIGENDIAN
1280 *d++ = (bgr&0x7C00)>>7;
1281 *d++ = (bgr&0x3E0)>>2;
1282 *d++ = (bgr&0x1F)<<3;
1284 *d++ = (bgr&0x1F)<<3;
1285 *d++ = (bgr&0x3E0)>>2;
1286 *d++ = (bgr&0x7C00)>>7;
1294 static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_size)
1296 const uint16_t *end;
1298 const uint16_t *mm_end;
1300 uint8_t *d = (uint8_t *)dst;
1301 const uint16_t *s = (uint16_t *)src;
1302 end = s + src_size/2;
1304 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
1305 __asm __volatile("pxor %%mm7,%%mm7\n\t":::"memory");
1311 "movq %1, %%mm0\n\t"
1312 "movq %1, %%mm1\n\t"
1313 "movq %1, %%mm2\n\t"
1314 "pand %2, %%mm0\n\t"
1315 "pand %3, %%mm1\n\t"
1316 "pand %4, %%mm2\n\t"
1317 "psllq $3, %%mm0\n\t"
1318 "psrlq $3, %%mm1\n\t"
1319 "psrlq $8, %%mm2\n\t"
1320 "movq %%mm0, %%mm3\n\t"
1321 "movq %%mm1, %%mm4\n\t"
1322 "movq %%mm2, %%mm5\n\t"
1323 "punpcklwd %%mm7, %%mm0\n\t"
1324 "punpcklwd %%mm7, %%mm1\n\t"
1325 "punpcklwd %%mm7, %%mm2\n\t"
1326 "punpckhwd %%mm7, %%mm3\n\t"
1327 "punpckhwd %%mm7, %%mm4\n\t"
1328 "punpckhwd %%mm7, %%mm5\n\t"
1329 "psllq $8, %%mm1\n\t"
1330 "psllq $16, %%mm2\n\t"
1331 "por %%mm1, %%mm0\n\t"
1332 "por %%mm2, %%mm0\n\t"
1333 "psllq $8, %%mm4\n\t"
1334 "psllq $16, %%mm5\n\t"
1335 "por %%mm4, %%mm3\n\t"
1336 "por %%mm5, %%mm3\n\t"
1337 MOVNTQ" %%mm0, %0\n\t"
1338 MOVNTQ" %%mm3, 8%0\n\t"
1340 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r)
1345 __asm __volatile(SFENCE:::"memory");
1346 __asm __volatile(EMMS:::"memory");
1350 register uint16_t bgr;
1352 #ifdef WORDS_BIGENDIAN
1354 *d++ = (bgr&0xF800)>>8;
1355 *d++ = (bgr&0x7E0)>>3;
1356 *d++ = (bgr&0x1F)<<3;
1358 *d++ = (bgr&0x1F)<<3;
1359 *d++ = (bgr&0x7E0)>>3;
1360 *d++ = (bgr&0xF800)>>8;
1366 static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, long src_size)
1369 /* TODO: unroll this loop */
1371 "xor %%"REG_a", %%"REG_a" \n\t"
1374 PREFETCH" 32(%0, %%"REG_a") \n\t"
1375 "movq (%0, %%"REG_a"), %%mm0 \n\t"
1376 "movq %%mm0, %%mm1 \n\t"
1377 "movq %%mm0, %%mm2 \n\t"
1378 "pslld $16, %%mm0 \n\t"
1379 "psrld $16, %%mm1 \n\t"
1380 "pand "MANGLE(mask32r)", %%mm0 \n\t"
1381 "pand "MANGLE(mask32g)", %%mm2 \n\t"
1382 "pand "MANGLE(mask32b)", %%mm1 \n\t"
1383 "por %%mm0, %%mm2 \n\t"
1384 "por %%mm1, %%mm2 \n\t"
1385 MOVNTQ" %%mm2, (%1, %%"REG_a") \n\t"
1386 "add $8, %%"REG_a" \n\t"
1387 "cmp %2, %%"REG_a" \n\t"
1389 :: "r" (src), "r"(dst), "r" (src_size-7)
1393 __asm __volatile(SFENCE:::"memory");
1394 __asm __volatile(EMMS:::"memory");
1397 unsigned num_pixels = src_size >> 2;
1398 for(i=0; i<num_pixels; i++)
1400 #ifdef WORDS_BIGENDIAN
1401 dst[4*i + 1] = src[4*i + 3];
1402 dst[4*i + 2] = src[4*i + 2];
1403 dst[4*i + 3] = src[4*i + 1];
1405 dst[4*i + 0] = src[4*i + 2];
1406 dst[4*i + 1] = src[4*i + 1];
1407 dst[4*i + 2] = src[4*i + 0];
1413 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, long src_size)
1417 long mmx_size= 23 - src_size;
1419 "movq "MANGLE(mask24r)", %%mm5 \n\t"
1420 "movq "MANGLE(mask24g)", %%mm6 \n\t"
1421 "movq "MANGLE(mask24b)", %%mm7 \n\t"
1424 PREFETCH" 32(%1, %%"REG_a") \n\t"
1425 "movq (%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1426 "movq (%1, %%"REG_a"), %%mm1 \n\t" // BGR BGR BG
1427 "movq 2(%1, %%"REG_a"), %%mm2 \n\t" // R BGR BGR B
1428 "psllq $16, %%mm0 \n\t" // 00 BGR BGR
1429 "pand %%mm5, %%mm0 \n\t"
1430 "pand %%mm6, %%mm1 \n\t"
1431 "pand %%mm7, %%mm2 \n\t"
1432 "por %%mm0, %%mm1 \n\t"
1433 "por %%mm2, %%mm1 \n\t"
1434 "movq 6(%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1435 MOVNTQ" %%mm1, (%2, %%"REG_a")\n\t" // RGB RGB RG
1436 "movq 8(%1, %%"REG_a"), %%mm1 \n\t" // R BGR BGR B
1437 "movq 10(%1, %%"REG_a"), %%mm2 \n\t" // GR BGR BGR
1438 "pand %%mm7, %%mm0 \n\t"
1439 "pand %%mm5, %%mm1 \n\t"
1440 "pand %%mm6, %%mm2 \n\t"
1441 "por %%mm0, %%mm1 \n\t"
1442 "por %%mm2, %%mm1 \n\t"
1443 "movq 14(%1, %%"REG_a"), %%mm0 \n\t" // R BGR BGR B
1444 MOVNTQ" %%mm1, 8(%2, %%"REG_a")\n\t" // B RGB RGB R
1445 "movq 16(%1, %%"REG_a"), %%mm1 \n\t" // GR BGR BGR
1446 "movq 18(%1, %%"REG_a"), %%mm2 \n\t" // BGR BGR BG
1447 "pand %%mm6, %%mm0 \n\t"
1448 "pand %%mm7, %%mm1 \n\t"
1449 "pand %%mm5, %%mm2 \n\t"
1450 "por %%mm0, %%mm1 \n\t"
1451 "por %%mm2, %%mm1 \n\t"
1452 MOVNTQ" %%mm1, 16(%2, %%"REG_a")\n\t"
1453 "add $24, %%"REG_a" \n\t"
1456 : "r" (src-mmx_size), "r"(dst-mmx_size)
1459 __asm __volatile(SFENCE:::"memory");
1460 __asm __volatile(EMMS:::"memory");
1462 if(mmx_size==23) return; //finihsed, was multiple of 8
1466 src_size= 23-mmx_size;
1470 for(i=0; i<src_size; i+=3)
1474 dst[i + 1] = src[i + 1];
1475 dst[i + 2] = src[i + 0];
1480 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1481 long width, long height,
1482 long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1485 const long chromWidth= width>>1;
1486 for(y=0; y<height; y++)
1489 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1491 "xor %%"REG_a", %%"REG_a" \n\t"
1494 PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1495 PREFETCH" 32(%2, %%"REG_a") \n\t"
1496 PREFETCH" 32(%3, %%"REG_a") \n\t"
1497 "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1498 "movq %%mm0, %%mm2 \n\t" // U(0)
1499 "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1500 "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1501 "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1503 "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1504 "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1505 "movq %%mm3, %%mm4 \n\t" // Y(0)
1506 "movq %%mm5, %%mm6 \n\t" // Y(8)
1507 "punpcklbw %%mm0, %%mm3 \n\t" // YUYV YUYV(0)
1508 "punpckhbw %%mm0, %%mm4 \n\t" // YUYV YUYV(4)
1509 "punpcklbw %%mm2, %%mm5 \n\t" // YUYV YUYV(8)
1510 "punpckhbw %%mm2, %%mm6 \n\t" // YUYV YUYV(12)
1512 MOVNTQ" %%mm3, (%0, %%"REG_a", 4)\n\t"
1513 MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1514 MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4)\n\t"
1515 MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1517 "add $8, %%"REG_a" \n\t"
1518 "cmp %4, %%"REG_a" \n\t"
1520 ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1525 #if defined ARCH_ALPHA && defined HAVE_MVI
1526 #define pl2yuy2(n) \
1531 asm("unpkbw %1, %0" : "=r"(y1) : "r"(y1)); \
1532 asm("unpkbw %1, %0" : "=r"(y2) : "r"(y2)); \
1533 asm("unpkbl %1, %0" : "=r"(u) : "r"(u)); \
1534 asm("unpkbl %1, %0" : "=r"(v) : "r"(v)); \
1535 yuv1 = (u << 8) + (v << 24); \
1542 uint64_t *qdst = (uint64_t *) dst;
1543 uint64_t *qdst2 = (uint64_t *) (dst + dstStride);
1544 const uint32_t *yc = (uint32_t *) ysrc;
1545 const uint32_t *yc2 = (uint32_t *) (ysrc + lumStride);
1546 const uint16_t *uc = (uint16_t*) usrc, *vc = (uint16_t*) vsrc;
1547 for(i = 0; i < chromWidth; i += 8){
1548 uint64_t y1, y2, yuv1, yuv2;
1551 asm("ldq $31,64(%0)" :: "r"(yc));
1552 asm("ldq $31,64(%0)" :: "r"(yc2));
1553 asm("ldq $31,64(%0)" :: "r"(uc));
1554 asm("ldq $31,64(%0)" :: "r"(vc));
1572 #elif __WORDSIZE >= 64
1574 uint64_t *ldst = (uint64_t *) dst;
1575 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1576 for(i = 0; i < chromWidth; i += 2){
1578 k = yc[0] + (uc[0] << 8) +
1579 (yc[1] << 16) + (vc[0] << 24);
1580 l = yc[2] + (uc[1] << 8) +
1581 (yc[3] << 16) + (vc[1] << 24);
1582 *ldst++ = k + (l << 32);
1589 int i, *idst = (int32_t *) dst;
1590 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1591 for(i = 0; i < chromWidth; i++){
1592 #ifdef WORDS_BIGENDIAN
1593 *idst++ = (yc[0] << 24)+ (uc[0] << 16) +
1594 (yc[1] << 8) + (vc[0] << 0);
1596 *idst++ = yc[0] + (uc[0] << 8) +
1597 (yc[1] << 16) + (vc[0] << 24);
1605 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1607 usrc += chromStride;
1608 vsrc += chromStride;
1622 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1623 * problem for anyone then tell me, and ill fix it)
1625 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1626 long width, long height,
1627 long lumStride, long chromStride, long dstStride)
1629 //FIXME interpolate chroma
1630 RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1633 static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1634 long width, long height,
1635 long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1638 const long chromWidth= width>>1;
1639 for(y=0; y<height; y++)
1642 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1644 "xor %%"REG_a", %%"REG_a" \n\t"
1647 PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1648 PREFETCH" 32(%2, %%"REG_a") \n\t"
1649 PREFETCH" 32(%3, %%"REG_a") \n\t"
1650 "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1651 "movq %%mm0, %%mm2 \n\t" // U(0)
1652 "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1653 "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1654 "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1656 "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1657 "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1658 "movq %%mm0, %%mm4 \n\t" // Y(0)
1659 "movq %%mm2, %%mm6 \n\t" // Y(8)
1660 "punpcklbw %%mm3, %%mm0 \n\t" // YUYV YUYV(0)
1661 "punpckhbw %%mm3, %%mm4 \n\t" // YUYV YUYV(4)
1662 "punpcklbw %%mm5, %%mm2 \n\t" // YUYV YUYV(8)
1663 "punpckhbw %%mm5, %%mm6 \n\t" // YUYV YUYV(12)
1665 MOVNTQ" %%mm0, (%0, %%"REG_a", 4)\n\t"
1666 MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1667 MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4)\n\t"
1668 MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1670 "add $8, %%"REG_a" \n\t"
1671 "cmp %4, %%"REG_a" \n\t"
1673 ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1677 //FIXME adapt the alpha asm code from yv12->yuy2
1679 #if __WORDSIZE >= 64
1681 uint64_t *ldst = (uint64_t *) dst;
1682 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1683 for(i = 0; i < chromWidth; i += 2){
1685 k = uc[0] + (yc[0] << 8) +
1686 (vc[0] << 16) + (yc[1] << 24);
1687 l = uc[1] + (yc[2] << 8) +
1688 (vc[1] << 16) + (yc[3] << 24);
1689 *ldst++ = k + (l << 32);
1696 int i, *idst = (int32_t *) dst;
1697 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1698 for(i = 0; i < chromWidth; i++){
1699 #ifdef WORDS_BIGENDIAN
1700 *idst++ = (uc[0] << 24)+ (yc[0] << 16) +
1701 (vc[0] << 8) + (yc[1] << 0);
1703 *idst++ = uc[0] + (yc[0] << 8) +
1704 (vc[0] << 16) + (yc[1] << 24);
1712 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1714 usrc += chromStride;
1715 vsrc += chromStride;
1729 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1730 * problem for anyone then tell me, and ill fix it)
1732 static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1733 long width, long height,
1734 long lumStride, long chromStride, long dstStride)
1736 //FIXME interpolate chroma
1737 RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1742 * width should be a multiple of 16
1744 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1745 long width, long height,
1746 long lumStride, long chromStride, long dstStride)
1748 RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1753 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1754 * problem for anyone then tell me, and ill fix it)
1756 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1757 long width, long height,
1758 long lumStride, long chromStride, long srcStride)
1761 const long chromWidth= width>>1;
1762 for(y=0; y<height; y+=2)
1766 "xor %%"REG_a", %%"REG_a" \n\t"
1767 "pcmpeqw %%mm7, %%mm7 \n\t"
1768 "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
1771 PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1772 "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1773 "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1774 "movq %%mm0, %%mm2 \n\t" // YUYV YUYV(0)
1775 "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(4)
1776 "psrlw $8, %%mm0 \n\t" // U0V0 U0V0(0)
1777 "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(4)
1778 "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
1779 "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
1780 "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1781 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
1783 MOVNTQ" %%mm2, (%1, %%"REG_a", 2)\n\t"
1785 "movq 16(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(8)
1786 "movq 24(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(12)
1787 "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(8)
1788 "movq %%mm2, %%mm4 \n\t" // YUYV YUYV(12)
1789 "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(8)
1790 "psrlw $8, %%mm2 \n\t" // U0V0 U0V0(12)
1791 "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
1792 "pand %%mm7, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
1793 "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
1794 "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
1796 MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2)\n\t"
1798 "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
1799 "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
1800 "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
1801 "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
1802 "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
1803 "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
1804 "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
1805 "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
1807 MOVNTQ" %%mm0, (%3, %%"REG_a") \n\t"
1808 MOVNTQ" %%mm2, (%2, %%"REG_a") \n\t"
1810 "add $8, %%"REG_a" \n\t"
1811 "cmp %4, %%"REG_a" \n\t"
1813 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1814 : "memory", "%"REG_a
1821 "xor %%"REG_a", %%"REG_a" \n\t"
1824 PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1825 "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1826 "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1827 "movq 16(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(8)
1828 "movq 24(%0, %%"REG_a", 4), %%mm3\n\t" // YUYV YUYV(12)
1829 "pand %%mm7, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
1830 "pand %%mm7, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
1831 "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
1832 "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
1833 "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
1834 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
1836 MOVNTQ" %%mm0, (%1, %%"REG_a", 2)\n\t"
1837 MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2)\n\t"
1839 "add $8, %%"REG_a" \n\t"
1840 "cmp %4, %%"REG_a" \n\t"
1843 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1844 : "memory", "%"REG_a
1848 for(i=0; i<chromWidth; i++)
1850 ydst[2*i+0] = src[4*i+0];
1851 udst[i] = src[4*i+1];
1852 ydst[2*i+1] = src[4*i+2];
1853 vdst[i] = src[4*i+3];
1858 for(i=0; i<chromWidth; i++)
1860 ydst[2*i+0] = src[4*i+0];
1861 ydst[2*i+1] = src[4*i+2];
1864 udst += chromStride;
1865 vdst += chromStride;
1870 asm volatile( EMMS" \n\t"
1876 static inline void RENAME(yvu9toyv12)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,
1877 uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1878 long width, long height, long lumStride, long chromStride)
1881 memcpy(ydst, ysrc, width*height);
1883 /* XXX: implement upscaling for U,V */
1886 static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, long srcWidth, long srcHeight, long srcStride, long dstStride)
1893 for(x=0; x<srcWidth-1; x++){
1894 dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1895 dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1897 dst[2*srcWidth-1]= src[srcWidth-1];
1901 for(y=1; y<srcHeight; y++){
1902 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
1903 const long mmxSize= srcWidth&~15;
1905 "mov %4, %%"REG_a" \n\t"
1907 "movq (%0, %%"REG_a"), %%mm0 \n\t"
1908 "movq (%1, %%"REG_a"), %%mm1 \n\t"
1909 "movq 1(%0, %%"REG_a"), %%mm2 \n\t"
1910 "movq 1(%1, %%"REG_a"), %%mm3 \n\t"
1911 "movq -1(%0, %%"REG_a"), %%mm4 \n\t"
1912 "movq -1(%1, %%"REG_a"), %%mm5 \n\t"
1913 PAVGB" %%mm0, %%mm5 \n\t"
1914 PAVGB" %%mm0, %%mm3 \n\t"
1915 PAVGB" %%mm0, %%mm5 \n\t"
1916 PAVGB" %%mm0, %%mm3 \n\t"
1917 PAVGB" %%mm1, %%mm4 \n\t"
1918 PAVGB" %%mm1, %%mm2 \n\t"
1919 PAVGB" %%mm1, %%mm4 \n\t"
1920 PAVGB" %%mm1, %%mm2 \n\t"
1921 "movq %%mm5, %%mm7 \n\t"
1922 "movq %%mm4, %%mm6 \n\t"
1923 "punpcklbw %%mm3, %%mm5 \n\t"
1924 "punpckhbw %%mm3, %%mm7 \n\t"
1925 "punpcklbw %%mm2, %%mm4 \n\t"
1926 "punpckhbw %%mm2, %%mm6 \n\t"
1928 MOVNTQ" %%mm5, (%2, %%"REG_a", 2)\n\t"
1929 MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1930 MOVNTQ" %%mm4, (%3, %%"REG_a", 2)\n\t"
1931 MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1933 "movq %%mm5, (%2, %%"REG_a", 2) \n\t"
1934 "movq %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1935 "movq %%mm4, (%3, %%"REG_a", 2) \n\t"
1936 "movq %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1938 "add $8, %%"REG_a" \n\t"
1940 :: "r" (src + mmxSize ), "r" (src + srcStride + mmxSize ),
1941 "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
1947 const long mmxSize=1;
1949 dst[0 ]= (3*src[0] + src[srcStride])>>2;
1950 dst[dstStride]= ( src[0] + 3*src[srcStride])>>2;
1952 for(x=mmxSize-1; x<srcWidth-1; x++){
1953 dst[2*x +1]= (3*src[x+0] + src[x+srcStride+1])>>2;
1954 dst[2*x+dstStride+2]= ( src[x+0] + 3*src[x+srcStride+1])>>2;
1955 dst[2*x+dstStride+1]= ( src[x+1] + 3*src[x+srcStride ])>>2;
1956 dst[2*x +2]= (3*src[x+1] + src[x+srcStride ])>>2;
1958 dst[srcWidth*2 -1 ]= (3*src[srcWidth-1] + src[srcWidth-1 + srcStride])>>2;
1959 dst[srcWidth*2 -1 + dstStride]= ( src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
1969 for(x=0; x<srcWidth-1; x++){
1970 dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1971 dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1973 dst[2*srcWidth-1]= src[srcWidth-1];
1975 for(x=0; x<srcWidth; x++){
1982 asm volatile( EMMS" \n\t"
1990 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1991 * problem for anyone then tell me, and ill fix it)
1992 * chrominance data is only taken from every secound line others are ignored FIXME write HQ version
1994 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1995 long width, long height,
1996 long lumStride, long chromStride, long srcStride)
1999 const long chromWidth= width>>1;
2000 for(y=0; y<height; y+=2)
2004 "xorl %%eax, %%eax \n\t"
2005 "pcmpeqw %%mm7, %%mm7 \n\t"
2006 "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
2009 PREFETCH" 64(%0, %%eax, 4) \n\t"
2010 "movq (%0, %%eax, 4), %%mm0 \n\t" // UYVY UYVY(0)
2011 "movq 8(%0, %%eax, 4), %%mm1 \n\t" // UYVY UYVY(4)
2012 "movq %%mm0, %%mm2 \n\t" // UYVY UYVY(0)
2013 "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(4)
2014 "pand %%mm7, %%mm0 \n\t" // U0V0 U0V0(0)
2015 "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(4)
2016 "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
2017 "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
2018 "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
2019 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
2021 MOVNTQ" %%mm2, (%1, %%eax, 2) \n\t"
2023 "movq 16(%0, %%eax, 4), %%mm1 \n\t" // UYVY UYVY(8)
2024 "movq 24(%0, %%eax, 4), %%mm2 \n\t" // UYVY UYVY(12)
2025 "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(8)
2026 "movq %%mm2, %%mm4 \n\t" // UYVY UYVY(12)
2027 "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(8)
2028 "pand %%mm7, %%mm2 \n\t" // U0V0 U0V0(12)
2029 "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
2030 "psrlw $8, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
2031 "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
2032 "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
2034 MOVNTQ" %%mm3, 8(%1, %%eax, 2) \n\t"
2036 "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
2037 "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
2038 "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
2039 "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
2040 "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
2041 "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
2042 "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
2043 "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
2045 MOVNTQ" %%mm0, (%3, %%eax) \n\t"
2046 MOVNTQ" %%mm2, (%2, %%eax) \n\t"
2048 "addl $8, %%eax \n\t"
2049 "cmpl %4, %%eax \n\t"
2051 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2059 "xorl %%eax, %%eax \n\t"
2062 PREFETCH" 64(%0, %%eax, 4) \n\t"
2063 "movq (%0, %%eax, 4), %%mm0 \n\t" // YUYV YUYV(0)
2064 "movq 8(%0, %%eax, 4), %%mm1 \n\t" // YUYV YUYV(4)
2065 "movq 16(%0, %%eax, 4), %%mm2 \n\t" // YUYV YUYV(8)
2066 "movq 24(%0, %%eax, 4), %%mm3 \n\t" // YUYV YUYV(12)
2067 "psrlw $8, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
2068 "psrlw $8, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
2069 "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
2070 "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
2071 "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
2072 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
2074 MOVNTQ" %%mm0, (%1, %%eax, 2) \n\t"
2075 MOVNTQ" %%mm2, 8(%1, %%eax, 2) \n\t"
2077 "addl $8, %%eax \n\t"
2078 "cmpl %4, %%eax \n\t"
2081 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2086 for(i=0; i<chromWidth; i++)
2088 udst[i] = src[4*i+0];
2089 ydst[2*i+0] = src[4*i+1];
2090 vdst[i] = src[4*i+2];
2091 ydst[2*i+1] = src[4*i+3];
2096 for(i=0; i<chromWidth; i++)
2098 ydst[2*i+0] = src[4*i+1];
2099 ydst[2*i+1] = src[4*i+3];
2102 udst += chromStride;
2103 vdst += chromStride;
2108 asm volatile( EMMS" \n\t"
2116 * height should be a multiple of 2 and width should be a multiple of 2 (if this is a
2117 * problem for anyone then tell me, and ill fix it)
2118 * chrominance data is only taken from every secound line others are ignored in the C version FIXME write HQ version
2120 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2121 long width, long height,
2122 long lumStride, long chromStride, long srcStride)
2125 const long chromWidth= width>>1;
2127 for(y=0; y<height-2; y+=2)
2133 "mov %2, %%"REG_a" \n\t"
2134 "movq "MANGLE(bgr2YCoeff)", %%mm6 \n\t"
2135 "movq "MANGLE(w1111)", %%mm5 \n\t"
2136 "pxor %%mm7, %%mm7 \n\t"
2137 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2140 PREFETCH" 64(%0, %%"REG_d") \n\t"
2141 "movd (%0, %%"REG_d"), %%mm0 \n\t"
2142 "movd 3(%0, %%"REG_d"), %%mm1 \n\t"
2143 "punpcklbw %%mm7, %%mm0 \n\t"
2144 "punpcklbw %%mm7, %%mm1 \n\t"
2145 "movd 6(%0, %%"REG_d"), %%mm2 \n\t"
2146 "movd 9(%0, %%"REG_d"), %%mm3 \n\t"
2147 "punpcklbw %%mm7, %%mm2 \n\t"
2148 "punpcklbw %%mm7, %%mm3 \n\t"
2149 "pmaddwd %%mm6, %%mm0 \n\t"
2150 "pmaddwd %%mm6, %%mm1 \n\t"
2151 "pmaddwd %%mm6, %%mm2 \n\t"
2152 "pmaddwd %%mm6, %%mm3 \n\t"
2153 #ifndef FAST_BGR2YV12
2154 "psrad $8, %%mm0 \n\t"
2155 "psrad $8, %%mm1 \n\t"
2156 "psrad $8, %%mm2 \n\t"
2157 "psrad $8, %%mm3 \n\t"
2159 "packssdw %%mm1, %%mm0 \n\t"
2160 "packssdw %%mm3, %%mm2 \n\t"
2161 "pmaddwd %%mm5, %%mm0 \n\t"
2162 "pmaddwd %%mm5, %%mm2 \n\t"
2163 "packssdw %%mm2, %%mm0 \n\t"
2164 "psraw $7, %%mm0 \n\t"
2166 "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
2167 "movd 15(%0, %%"REG_d"), %%mm1 \n\t"
2168 "punpcklbw %%mm7, %%mm4 \n\t"
2169 "punpcklbw %%mm7, %%mm1 \n\t"
2170 "movd 18(%0, %%"REG_d"), %%mm2 \n\t"
2171 "movd 21(%0, %%"REG_d"), %%mm3 \n\t"
2172 "punpcklbw %%mm7, %%mm2 \n\t"
2173 "punpcklbw %%mm7, %%mm3 \n\t"
2174 "pmaddwd %%mm6, %%mm4 \n\t"
2175 "pmaddwd %%mm6, %%mm1 \n\t"
2176 "pmaddwd %%mm6, %%mm2 \n\t"
2177 "pmaddwd %%mm6, %%mm3 \n\t"
2178 #ifndef FAST_BGR2YV12
2179 "psrad $8, %%mm4 \n\t"
2180 "psrad $8, %%mm1 \n\t"
2181 "psrad $8, %%mm2 \n\t"
2182 "psrad $8, %%mm3 \n\t"
2184 "packssdw %%mm1, %%mm4 \n\t"
2185 "packssdw %%mm3, %%mm2 \n\t"
2186 "pmaddwd %%mm5, %%mm4 \n\t"
2187 "pmaddwd %%mm5, %%mm2 \n\t"
2188 "add $24, %%"REG_d" \n\t"
2189 "packssdw %%mm2, %%mm4 \n\t"
2190 "psraw $7, %%mm4 \n\t"
2192 "packuswb %%mm4, %%mm0 \n\t"
2193 "paddusb "MANGLE(bgr2YOffset)", %%mm0 \n\t"
2195 MOVNTQ" %%mm0, (%1, %%"REG_a") \n\t"
2196 "add $8, %%"REG_a" \n\t"
2198 : : "r" (src+width*3), "r" (ydst+width), "g" (-width)
2199 : "%"REG_a, "%"REG_d
2206 "mov %4, %%"REG_a" \n\t"
2207 "movq "MANGLE(w1111)", %%mm5 \n\t"
2208 "movq "MANGLE(bgr2UCoeff)", %%mm6 \n\t"
2209 "pxor %%mm7, %%mm7 \n\t"
2210 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2211 "add %%"REG_d", %%"REG_d" \n\t"
2214 PREFETCH" 64(%0, %%"REG_d") \n\t"
2215 PREFETCH" 64(%1, %%"REG_d") \n\t"
2216 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2217 "movq (%0, %%"REG_d"), %%mm0 \n\t"
2218 "movq (%1, %%"REG_d"), %%mm1 \n\t"
2219 "movq 6(%0, %%"REG_d"), %%mm2 \n\t"
2220 "movq 6(%1, %%"REG_d"), %%mm3 \n\t"
2221 PAVGB" %%mm1, %%mm0 \n\t"
2222 PAVGB" %%mm3, %%mm2 \n\t"
2223 "movq %%mm0, %%mm1 \n\t"
2224 "movq %%mm2, %%mm3 \n\t"
2225 "psrlq $24, %%mm0 \n\t"
2226 "psrlq $24, %%mm2 \n\t"
2227 PAVGB" %%mm1, %%mm0 \n\t"
2228 PAVGB" %%mm3, %%mm2 \n\t"
2229 "punpcklbw %%mm7, %%mm0 \n\t"
2230 "punpcklbw %%mm7, %%mm2 \n\t"
2232 "movd (%0, %%"REG_d"), %%mm0 \n\t"
2233 "movd (%1, %%"REG_d"), %%mm1 \n\t"
2234 "movd 3(%0, %%"REG_d"), %%mm2 \n\t"
2235 "movd 3(%1, %%"REG_d"), %%mm3 \n\t"
2236 "punpcklbw %%mm7, %%mm0 \n\t"
2237 "punpcklbw %%mm7, %%mm1 \n\t"
2238 "punpcklbw %%mm7, %%mm2 \n\t"
2239 "punpcklbw %%mm7, %%mm3 \n\t"
2240 "paddw %%mm1, %%mm0 \n\t"
2241 "paddw %%mm3, %%mm2 \n\t"
2242 "paddw %%mm2, %%mm0 \n\t"
2243 "movd 6(%0, %%"REG_d"), %%mm4 \n\t"
2244 "movd 6(%1, %%"REG_d"), %%mm1 \n\t"
2245 "movd 9(%0, %%"REG_d"), %%mm2 \n\t"
2246 "movd 9(%1, %%"REG_d"), %%mm3 \n\t"
2247 "punpcklbw %%mm7, %%mm4 \n\t"
2248 "punpcklbw %%mm7, %%mm1 \n\t"
2249 "punpcklbw %%mm7, %%mm2 \n\t"
2250 "punpcklbw %%mm7, %%mm3 \n\t"
2251 "paddw %%mm1, %%mm4 \n\t"
2252 "paddw %%mm3, %%mm2 \n\t"
2253 "paddw %%mm4, %%mm2 \n\t"
2254 "psrlw $2, %%mm0 \n\t"
2255 "psrlw $2, %%mm2 \n\t"
2257 "movq "MANGLE(bgr2VCoeff)", %%mm1 \n\t"
2258 "movq "MANGLE(bgr2VCoeff)", %%mm3 \n\t"
2260 "pmaddwd %%mm0, %%mm1 \n\t"
2261 "pmaddwd %%mm2, %%mm3 \n\t"
2262 "pmaddwd %%mm6, %%mm0 \n\t"
2263 "pmaddwd %%mm6, %%mm2 \n\t"
2264 #ifndef FAST_BGR2YV12
2265 "psrad $8, %%mm0 \n\t"
2266 "psrad $8, %%mm1 \n\t"
2267 "psrad $8, %%mm2 \n\t"
2268 "psrad $8, %%mm3 \n\t"
2270 "packssdw %%mm2, %%mm0 \n\t"
2271 "packssdw %%mm3, %%mm1 \n\t"
2272 "pmaddwd %%mm5, %%mm0 \n\t"
2273 "pmaddwd %%mm5, %%mm1 \n\t"
2274 "packssdw %%mm1, %%mm0 \n\t" // V1 V0 U1 U0
2275 "psraw $7, %%mm0 \n\t"
2277 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2278 "movq 12(%0, %%"REG_d"), %%mm4 \n\t"
2279 "movq 12(%1, %%"REG_d"), %%mm1 \n\t"
2280 "movq 18(%0, %%"REG_d"), %%mm2 \n\t"
2281 "movq 18(%1, %%"REG_d"), %%mm3 \n\t"
2282 PAVGB" %%mm1, %%mm4 \n\t"
2283 PAVGB" %%mm3, %%mm2 \n\t"
2284 "movq %%mm4, %%mm1 \n\t"
2285 "movq %%mm2, %%mm3 \n\t"
2286 "psrlq $24, %%mm4 \n\t"
2287 "psrlq $24, %%mm2 \n\t"
2288 PAVGB" %%mm1, %%mm4 \n\t"
2289 PAVGB" %%mm3, %%mm2 \n\t"
2290 "punpcklbw %%mm7, %%mm4 \n\t"
2291 "punpcklbw %%mm7, %%mm2 \n\t"
2293 "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
2294 "movd 12(%1, %%"REG_d"), %%mm1 \n\t"
2295 "movd 15(%0, %%"REG_d"), %%mm2 \n\t"
2296 "movd 15(%1, %%"REG_d"), %%mm3 \n\t"
2297 "punpcklbw %%mm7, %%mm4 \n\t"
2298 "punpcklbw %%mm7, %%mm1 \n\t"
2299 "punpcklbw %%mm7, %%mm2 \n\t"
2300 "punpcklbw %%mm7, %%mm3 \n\t"
2301 "paddw %%mm1, %%mm4 \n\t"
2302 "paddw %%mm3, %%mm2 \n\t"
2303 "paddw %%mm2, %%mm4 \n\t"
2304 "movd 18(%0, %%"REG_d"), %%mm5 \n\t"
2305 "movd 18(%1, %%"REG_d"), %%mm1 \n\t"
2306 "movd 21(%0, %%"REG_d"), %%mm2 \n\t"
2307 "movd 21(%1, %%"REG_d"), %%mm3 \n\t"
2308 "punpcklbw %%mm7, %%mm5 \n\t"
2309 "punpcklbw %%mm7, %%mm1 \n\t"
2310 "punpcklbw %%mm7, %%mm2 \n\t"
2311 "punpcklbw %%mm7, %%mm3 \n\t"
2312 "paddw %%mm1, %%mm5 \n\t"
2313 "paddw %%mm3, %%mm2 \n\t"
2314 "paddw %%mm5, %%mm2 \n\t"
2315 "movq "MANGLE(w1111)", %%mm5 \n\t"
2316 "psrlw $2, %%mm4 \n\t"
2317 "psrlw $2, %%mm2 \n\t"
2319 "movq "MANGLE(bgr2VCoeff)", %%mm1 \n\t"
2320 "movq "MANGLE(bgr2VCoeff)", %%mm3 \n\t"
2322 "pmaddwd %%mm4, %%mm1 \n\t"
2323 "pmaddwd %%mm2, %%mm3 \n\t"
2324 "pmaddwd %%mm6, %%mm4 \n\t"
2325 "pmaddwd %%mm6, %%mm2 \n\t"
2326 #ifndef FAST_BGR2YV12
2327 "psrad $8, %%mm4 \n\t"
2328 "psrad $8, %%mm1 \n\t"
2329 "psrad $8, %%mm2 \n\t"
2330 "psrad $8, %%mm3 \n\t"
2332 "packssdw %%mm2, %%mm4 \n\t"
2333 "packssdw %%mm3, %%mm1 \n\t"
2334 "pmaddwd %%mm5, %%mm4 \n\t"
2335 "pmaddwd %%mm5, %%mm1 \n\t"
2336 "add $24, %%"REG_d" \n\t"
2337 "packssdw %%mm1, %%mm4 \n\t" // V3 V2 U3 U2
2338 "psraw $7, %%mm4 \n\t"
2340 "movq %%mm0, %%mm1 \n\t"
2341 "punpckldq %%mm4, %%mm0 \n\t"
2342 "punpckhdq %%mm4, %%mm1 \n\t"
2343 "packsswb %%mm1, %%mm0 \n\t"
2344 "paddb "MANGLE(bgr2UVOffset)", %%mm0 \n\t"
2345 "movd %%mm0, (%2, %%"REG_a") \n\t"
2346 "punpckhdq %%mm0, %%mm0 \n\t"
2347 "movd %%mm0, (%3, %%"REG_a") \n\t"
2348 "add $4, %%"REG_a" \n\t"
2350 : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth)
2351 : "%"REG_a, "%"REG_d
2354 udst += chromStride;
2355 vdst += chromStride;
2359 asm volatile( EMMS" \n\t"
2365 for(; y<height; y+=2)
2368 for(i=0; i<chromWidth; i++)
2370 unsigned int b= src[6*i+0];
2371 unsigned int g= src[6*i+1];
2372 unsigned int r= src[6*i+2];
2374 unsigned int Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2375 unsigned int V = ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128;
2376 unsigned int U = ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128;
2386 Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2392 for(i=0; i<chromWidth; i++)
2394 unsigned int b= src[6*i+0];
2395 unsigned int g= src[6*i+1];
2396 unsigned int r= src[6*i+2];
2398 unsigned int Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2406 Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2409 udst += chromStride;
2410 vdst += chromStride;
2416 void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest,
2417 long width, long height, long src1Stride,
2418 long src2Stride, long dstStride){
2421 for(h=0; h < height; h++)
2428 "xor %%"REG_a", %%"REG_a" \n\t"
2430 PREFETCH" 64(%1, %%"REG_a") \n\t"
2431 PREFETCH" 64(%2, %%"REG_a") \n\t"
2432 "movdqa (%1, %%"REG_a"), %%xmm0 \n\t"
2433 "movdqa (%1, %%"REG_a"), %%xmm1 \n\t"
2434 "movdqa (%2, %%"REG_a"), %%xmm2 \n\t"
2435 "punpcklbw %%xmm2, %%xmm0 \n\t"
2436 "punpckhbw %%xmm2, %%xmm1 \n\t"
2437 "movntdq %%xmm0, (%0, %%"REG_a", 2)\n\t"
2438 "movntdq %%xmm1, 16(%0, %%"REG_a", 2)\n\t"
2439 "add $16, %%"REG_a" \n\t"
2440 "cmp %3, %%"REG_a" \n\t"
2442 ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2443 : "memory", "%"REG_a""
2447 "xor %%"REG_a", %%"REG_a" \n\t"
2449 PREFETCH" 64(%1, %%"REG_a") \n\t"
2450 PREFETCH" 64(%2, %%"REG_a") \n\t"
2451 "movq (%1, %%"REG_a"), %%mm0 \n\t"
2452 "movq 8(%1, %%"REG_a"), %%mm2 \n\t"
2453 "movq %%mm0, %%mm1 \n\t"
2454 "movq %%mm2, %%mm3 \n\t"
2455 "movq (%2, %%"REG_a"), %%mm4 \n\t"
2456 "movq 8(%2, %%"REG_a"), %%mm5 \n\t"
2457 "punpcklbw %%mm4, %%mm0 \n\t"
2458 "punpckhbw %%mm4, %%mm1 \n\t"
2459 "punpcklbw %%mm5, %%mm2 \n\t"
2460 "punpckhbw %%mm5, %%mm3 \n\t"
2461 MOVNTQ" %%mm0, (%0, %%"REG_a", 2)\n\t"
2462 MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2)\n\t"
2463 MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2)\n\t"
2464 MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2)\n\t"
2465 "add $16, %%"REG_a" \n\t"
2466 "cmp %3, %%"REG_a" \n\t"
2468 ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2469 : "memory", "%"REG_a
2472 for(w= (width&(~15)); w < width; w++)
2474 dest[2*w+0] = src1[w];
2475 dest[2*w+1] = src2[w];
2478 for(w=0; w < width; w++)
2480 dest[2*w+0] = src1[w];
2481 dest[2*w+1] = src2[w];
2497 static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
2498 uint8_t *dst1, uint8_t *dst2,
2499 long width, long height,
2500 long srcStride1, long srcStride2,
2501 long dstStride1, long dstStride2)
2504 w=width/2; h=height/2;
2509 ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
2512 const uint8_t* s1=src1+srcStride1*(y>>1);
2513 uint8_t* d=dst1+dstStride1*y;
2520 "movq %1, %%mm0\n\t"
2521 "movq 8%1, %%mm2\n\t"
2522 "movq 16%1, %%mm4\n\t"
2523 "movq 24%1, %%mm6\n\t"
2524 "movq %%mm0, %%mm1\n\t"
2525 "movq %%mm2, %%mm3\n\t"
2526 "movq %%mm4, %%mm5\n\t"
2527 "movq %%mm6, %%mm7\n\t"
2528 "punpcklbw %%mm0, %%mm0\n\t"
2529 "punpckhbw %%mm1, %%mm1\n\t"
2530 "punpcklbw %%mm2, %%mm2\n\t"
2531 "punpckhbw %%mm3, %%mm3\n\t"
2532 "punpcklbw %%mm4, %%mm4\n\t"
2533 "punpckhbw %%mm5, %%mm5\n\t"
2534 "punpcklbw %%mm6, %%mm6\n\t"
2535 "punpckhbw %%mm7, %%mm7\n\t"
2536 MOVNTQ" %%mm0, %0\n\t"
2537 MOVNTQ" %%mm1, 8%0\n\t"
2538 MOVNTQ" %%mm2, 16%0\n\t"
2539 MOVNTQ" %%mm3, 24%0\n\t"
2540 MOVNTQ" %%mm4, 32%0\n\t"
2541 MOVNTQ" %%mm5, 40%0\n\t"
2542 MOVNTQ" %%mm6, 48%0\n\t"
2543 MOVNTQ" %%mm7, 56%0"
2549 for(;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
2552 const uint8_t* s2=src2+srcStride2*(y>>1);
2553 uint8_t* d=dst2+dstStride2*y;
2560 "movq %1, %%mm0\n\t"
2561 "movq 8%1, %%mm2\n\t"
2562 "movq 16%1, %%mm4\n\t"
2563 "movq 24%1, %%mm6\n\t"
2564 "movq %%mm0, %%mm1\n\t"
2565 "movq %%mm2, %%mm3\n\t"
2566 "movq %%mm4, %%mm5\n\t"
2567 "movq %%mm6, %%mm7\n\t"
2568 "punpcklbw %%mm0, %%mm0\n\t"
2569 "punpckhbw %%mm1, %%mm1\n\t"
2570 "punpcklbw %%mm2, %%mm2\n\t"
2571 "punpckhbw %%mm3, %%mm3\n\t"
2572 "punpcklbw %%mm4, %%mm4\n\t"
2573 "punpckhbw %%mm5, %%mm5\n\t"
2574 "punpcklbw %%mm6, %%mm6\n\t"
2575 "punpckhbw %%mm7, %%mm7\n\t"
2576 MOVNTQ" %%mm0, %0\n\t"
2577 MOVNTQ" %%mm1, 8%0\n\t"
2578 MOVNTQ" %%mm2, 16%0\n\t"
2579 MOVNTQ" %%mm3, 24%0\n\t"
2580 MOVNTQ" %%mm4, 32%0\n\t"
2581 MOVNTQ" %%mm5, 40%0\n\t"
2582 MOVNTQ" %%mm6, 48%0\n\t"
2583 MOVNTQ" %%mm7, 56%0"
2589 for(;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
2600 static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
2602 long width, long height,
2603 long srcStride1, long srcStride2,
2604 long srcStride3, long dstStride)
2607 w=width/2; h=height;
2609 const uint8_t* yp=src1+srcStride1*y;
2610 const uint8_t* up=src2+srcStride2*(y>>2);
2611 const uint8_t* vp=src3+srcStride3*(y>>2);
2612 uint8_t* d=dst+dstStride*y;
2618 PREFETCH" 32(%1, %0)\n\t"
2619 PREFETCH" 32(%2, %0)\n\t"
2620 PREFETCH" 32(%3, %0)\n\t"
2621 "movq (%1, %0, 4), %%mm0\n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2622 "movq (%2, %0), %%mm1\n\t" /* U0U1U2U3U4U5U6U7 */
2623 "movq (%3, %0), %%mm2\n\t" /* V0V1V2V3V4V5V6V7 */
2624 "movq %%mm0, %%mm3\n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2625 "movq %%mm1, %%mm4\n\t" /* U0U1U2U3U4U5U6U7 */
2626 "movq %%mm2, %%mm5\n\t" /* V0V1V2V3V4V5V6V7 */
2627 "punpcklbw %%mm1, %%mm1\n\t" /* U0U0 U1U1 U2U2 U3U3 */
2628 "punpcklbw %%mm2, %%mm2\n\t" /* V0V0 V1V1 V2V2 V3V3 */
2629 "punpckhbw %%mm4, %%mm4\n\t" /* U4U4 U5U5 U6U6 U7U7 */
2630 "punpckhbw %%mm5, %%mm5\n\t" /* V4V4 V5V5 V6V6 V7V7 */
2632 "movq %%mm1, %%mm6\n\t"
2633 "punpcklbw %%mm2, %%mm1\n\t" /* U0V0 U0V0 U1V1 U1V1*/
2634 "punpcklbw %%mm1, %%mm0\n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
2635 "punpckhbw %%mm1, %%mm3\n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
2636 MOVNTQ" %%mm0, (%4, %0, 8)\n\t"
2637 MOVNTQ" %%mm3, 8(%4, %0, 8)\n\t"
2639 "punpckhbw %%mm2, %%mm6\n\t" /* U2V2 U2V2 U3V3 U3V3*/
2640 "movq 8(%1, %0, 4), %%mm0\n\t"
2641 "movq %%mm0, %%mm3\n\t"
2642 "punpcklbw %%mm6, %%mm0\n\t" /* Y U2 Y V2 Y U2 Y V2*/
2643 "punpckhbw %%mm6, %%mm3\n\t" /* Y U3 Y V3 Y U3 Y V3*/
2644 MOVNTQ" %%mm0, 16(%4, %0, 8)\n\t"
2645 MOVNTQ" %%mm3, 24(%4, %0, 8)\n\t"
2647 "movq %%mm4, %%mm6\n\t"
2648 "movq 16(%1, %0, 4), %%mm0\n\t"
2649 "movq %%mm0, %%mm3\n\t"
2650 "punpcklbw %%mm5, %%mm4\n\t"
2651 "punpcklbw %%mm4, %%mm0\n\t" /* Y U4 Y V4 Y U4 Y V4*/
2652 "punpckhbw %%mm4, %%mm3\n\t" /* Y U5 Y V5 Y U5 Y V5*/
2653 MOVNTQ" %%mm0, 32(%4, %0, 8)\n\t"
2654 MOVNTQ" %%mm3, 40(%4, %0, 8)\n\t"
2656 "punpckhbw %%mm5, %%mm6\n\t"
2657 "movq 24(%1, %0, 4), %%mm0\n\t"
2658 "movq %%mm0, %%mm3\n\t"
2659 "punpcklbw %%mm6, %%mm0\n\t" /* Y U6 Y V6 Y U6 Y V6*/
2660 "punpckhbw %%mm6, %%mm3\n\t" /* Y U7 Y V7 Y U7 Y V7*/
2661 MOVNTQ" %%mm0, 48(%4, %0, 8)\n\t"
2662 MOVNTQ" %%mm3, 56(%4, %0, 8)\n\t"
2665 : "r"(yp), "r" (up), "r"(vp), "r"(d)
2671 const long x2= x<<2;