3 * rgb2rgb.c, Software RGB to RGB convertor
4 * pluralize by Software PAL8 to RGB convertor
5 * Software YUV to YUV convertor
6 * Software YUV to RGB convertor
7 * Written by Nick Kurshev.
8 * palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
9 * lot of big-endian byteorder fixes by Alex Beregszaszi
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
27 #include <inttypes.h> /* for __WORDSIZE */
30 // #warning You have misconfigured system and probably will lose performance!
31 #define __WORDSIZE MP_WORDSIZE
49 #define PREFETCH "prefetch"
50 #define PREFETCHW "prefetchw"
51 #define PAVGB "pavgusb"
52 #elif defined ( HAVE_MMX2 )
53 #define PREFETCH "prefetchnta"
54 #define PREFETCHW "prefetcht0"
61 #define PREFETCH "/nop"
62 #define PREFETCHW "/nop"
67 /* On K6 femms is faster of emms. On K7 femms is directly mapped on emms. */
74 #define MOVNTQ "movntq"
75 #define SFENCE "sfence"
85 static inline void RENAME(rgb24to32)(const uint8_t *src,uint8_t *dst,long src_size)
88 const uint8_t *s = src;
91 const uint8_t *mm_end;
95 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
97 __asm __volatile("movq %0, %%mm7"::"m"(mask32):"memory");
103 "punpckldq 3%1, %%mm0\n\t"
104 "movd 6%1, %%mm1\n\t"
105 "punpckldq 9%1, %%mm1\n\t"
106 "movd 12%1, %%mm2\n\t"
107 "punpckldq 15%1, %%mm2\n\t"
108 "movd 18%1, %%mm3\n\t"
109 "punpckldq 21%1, %%mm3\n\t"
110 "pand %%mm7, %%mm0\n\t"
111 "pand %%mm7, %%mm1\n\t"
112 "pand %%mm7, %%mm2\n\t"
113 "pand %%mm7, %%mm3\n\t"
114 MOVNTQ" %%mm0, %0\n\t"
115 MOVNTQ" %%mm1, 8%0\n\t"
116 MOVNTQ" %%mm2, 16%0\n\t"
124 __asm __volatile(SFENCE:::"memory");
125 __asm __volatile(EMMS:::"memory");
129 #ifdef WORDS_BIGENDIAN
130 /* RGB24 (= R,G,B) -> RGB32 (= A,B,G,R) */
145 static inline void RENAME(rgb32to24)(const uint8_t *src,uint8_t *dst,long src_size)
148 const uint8_t *s = src;
151 const uint8_t *mm_end;
155 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
162 "movq 8%1, %%mm1\n\t"
163 "movq 16%1, %%mm4\n\t"
164 "movq 24%1, %%mm5\n\t"
165 "movq %%mm0, %%mm2\n\t"
166 "movq %%mm1, %%mm3\n\t"
167 "movq %%mm4, %%mm6\n\t"
168 "movq %%mm5, %%mm7\n\t"
169 "psrlq $8, %%mm2\n\t"
170 "psrlq $8, %%mm3\n\t"
171 "psrlq $8, %%mm6\n\t"
172 "psrlq $8, %%mm7\n\t"
181 "por %%mm2, %%mm0\n\t"
182 "por %%mm3, %%mm1\n\t"
183 "por %%mm6, %%mm4\n\t"
184 "por %%mm7, %%mm5\n\t"
186 "movq %%mm1, %%mm2\n\t"
187 "movq %%mm4, %%mm3\n\t"
188 "psllq $48, %%mm2\n\t"
189 "psllq $32, %%mm3\n\t"
192 "por %%mm2, %%mm0\n\t"
193 "psrlq $16, %%mm1\n\t"
194 "psrlq $32, %%mm4\n\t"
195 "psllq $16, %%mm5\n\t"
196 "por %%mm3, %%mm1\n\t"
198 "por %%mm5, %%mm4\n\t"
200 MOVNTQ" %%mm0, %0\n\t"
201 MOVNTQ" %%mm1, 8%0\n\t"
204 :"m"(*s),"m"(mask24l),
205 "m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
210 __asm __volatile(SFENCE:::"memory");
211 __asm __volatile(EMMS:::"memory");
215 #ifdef WORDS_BIGENDIAN
216 /* RGB32 (= A,B,G,R) -> RGB24 (= R,G,B) */
232 Original by Strepto/Astral
233 ported to gcc & bugfixed : A'rpi
234 MMX2, 3DNOW optimization by Nick Kurshev
235 32bit c version, and and&add trick by Michael Niedermayer
237 static inline void RENAME(rgb15to16)(const uint8_t *src,uint8_t *dst,long src_size)
239 register const uint8_t* s=src;
240 register uint8_t* d=dst;
241 register const uint8_t *end;
242 const uint8_t *mm_end;
245 __asm __volatile(PREFETCH" %0"::"m"(*s));
246 __asm __volatile("movq %0, %%mm4"::"m"(mask15s));
253 "movq 8%1, %%mm2\n\t"
254 "movq %%mm0, %%mm1\n\t"
255 "movq %%mm2, %%mm3\n\t"
256 "pand %%mm4, %%mm0\n\t"
257 "pand %%mm4, %%mm2\n\t"
258 "paddw %%mm1, %%mm0\n\t"
259 "paddw %%mm3, %%mm2\n\t"
260 MOVNTQ" %%mm0, %0\n\t"
268 __asm __volatile(SFENCE:::"memory");
269 __asm __volatile(EMMS:::"memory");
274 register unsigned x= *((uint32_t *)s);
275 *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
281 register unsigned short x= *((uint16_t *)s);
282 *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
286 static inline void RENAME(rgb16to15)(const uint8_t *src,uint8_t *dst,long src_size)
288 register const uint8_t* s=src;
289 register uint8_t* d=dst;
290 register const uint8_t *end;
291 const uint8_t *mm_end;
294 __asm __volatile(PREFETCH" %0"::"m"(*s));
295 __asm __volatile("movq %0, %%mm7"::"m"(mask15rg));
296 __asm __volatile("movq %0, %%mm6"::"m"(mask15b));
303 "movq 8%1, %%mm2\n\t"
304 "movq %%mm0, %%mm1\n\t"
305 "movq %%mm2, %%mm3\n\t"
306 "psrlq $1, %%mm0\n\t"
307 "psrlq $1, %%mm2\n\t"
308 "pand %%mm7, %%mm0\n\t"
309 "pand %%mm7, %%mm2\n\t"
310 "pand %%mm6, %%mm1\n\t"
311 "pand %%mm6, %%mm3\n\t"
312 "por %%mm1, %%mm0\n\t"
313 "por %%mm3, %%mm2\n\t"
314 MOVNTQ" %%mm0, %0\n\t"
322 __asm __volatile(SFENCE:::"memory");
323 __asm __volatile(EMMS:::"memory");
328 register uint32_t x= *((uint32_t *)s);
329 *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
335 register uint16_t x= *((uint16_t *)s);
336 *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
342 static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, long src_size)
344 const uint8_t *s = src;
347 const uint8_t *mm_end;
349 uint16_t *d = (uint16_t *)dst;
353 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
355 "movq %3, %%mm5 \n\t"
356 "movq %4, %%mm6 \n\t"
357 "movq %5, %%mm7 \n\t"
360 PREFETCH" 32(%1) \n\t"
361 "movd (%1), %%mm0 \n\t"
362 "movd 4(%1), %%mm3 \n\t"
363 "punpckldq 8(%1), %%mm0 \n\t"
364 "punpckldq 12(%1), %%mm3 \n\t"
365 "movq %%mm0, %%mm1 \n\t"
366 "movq %%mm3, %%mm4 \n\t"
367 "pand %%mm6, %%mm0 \n\t"
368 "pand %%mm6, %%mm3 \n\t"
369 "pmaddwd %%mm7, %%mm0 \n\t"
370 "pmaddwd %%mm7, %%mm3 \n\t"
371 "pand %%mm5, %%mm1 \n\t"
372 "pand %%mm5, %%mm4 \n\t"
373 "por %%mm1, %%mm0 \n\t"
374 "por %%mm4, %%mm3 \n\t"
375 "psrld $5, %%mm0 \n\t"
376 "pslld $11, %%mm3 \n\t"
377 "por %%mm3, %%mm0 \n\t"
378 MOVNTQ" %%mm0, (%0) \n\t"
384 : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
387 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
391 ::"m"(red_16mask),"m"(green_16mask));
397 "movd 4%1, %%mm3\n\t"
398 "punpckldq 8%1, %%mm0\n\t"
399 "punpckldq 12%1, %%mm3\n\t"
400 "movq %%mm0, %%mm1\n\t"
401 "movq %%mm0, %%mm2\n\t"
402 "movq %%mm3, %%mm4\n\t"
403 "movq %%mm3, %%mm5\n\t"
404 "psrlq $3, %%mm0\n\t"
405 "psrlq $3, %%mm3\n\t"
408 "psrlq $5, %%mm1\n\t"
409 "psrlq $5, %%mm4\n\t"
410 "pand %%mm6, %%mm1\n\t"
411 "pand %%mm6, %%mm4\n\t"
412 "psrlq $8, %%mm2\n\t"
413 "psrlq $8, %%mm5\n\t"
414 "pand %%mm7, %%mm2\n\t"
415 "pand %%mm7, %%mm5\n\t"
416 "por %%mm1, %%mm0\n\t"
417 "por %%mm4, %%mm3\n\t"
418 "por %%mm2, %%mm0\n\t"
419 "por %%mm5, %%mm3\n\t"
420 "psllq $16, %%mm3\n\t"
421 "por %%mm3, %%mm0\n\t"
422 MOVNTQ" %%mm0, %0\n\t"
423 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
428 __asm __volatile(SFENCE:::"memory");
429 __asm __volatile(EMMS:::"memory");
433 register int rgb = *(uint32_t*)s; s += 4;
434 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
438 static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
440 const uint8_t *s = src;
443 const uint8_t *mm_end;
445 uint16_t *d = (uint16_t *)dst;
448 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
452 ::"m"(red_16mask),"m"(green_16mask));
459 "movd 4%1, %%mm3\n\t"
460 "punpckldq 8%1, %%mm0\n\t"
461 "punpckldq 12%1, %%mm3\n\t"
462 "movq %%mm0, %%mm1\n\t"
463 "movq %%mm0, %%mm2\n\t"
464 "movq %%mm3, %%mm4\n\t"
465 "movq %%mm3, %%mm5\n\t"
466 "psllq $8, %%mm0\n\t"
467 "psllq $8, %%mm3\n\t"
468 "pand %%mm7, %%mm0\n\t"
469 "pand %%mm7, %%mm3\n\t"
470 "psrlq $5, %%mm1\n\t"
471 "psrlq $5, %%mm4\n\t"
472 "pand %%mm6, %%mm1\n\t"
473 "pand %%mm6, %%mm4\n\t"
474 "psrlq $19, %%mm2\n\t"
475 "psrlq $19, %%mm5\n\t"
478 "por %%mm1, %%mm0\n\t"
479 "por %%mm4, %%mm3\n\t"
480 "por %%mm2, %%mm0\n\t"
481 "por %%mm5, %%mm3\n\t"
482 "psllq $16, %%mm3\n\t"
483 "por %%mm3, %%mm0\n\t"
484 MOVNTQ" %%mm0, %0\n\t"
485 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
489 __asm __volatile(SFENCE:::"memory");
490 __asm __volatile(EMMS:::"memory");
494 register int rgb = *(uint32_t*)s; s += 4;
495 *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
499 static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, long src_size)
501 const uint8_t *s = src;
504 const uint8_t *mm_end;
506 uint16_t *d = (uint16_t *)dst;
510 #if 1 //is faster only if multiplies are reasonable fast (FIXME figure out on which cpus this is faster, on Athlon its slightly faster)
512 "movq %3, %%mm5 \n\t"
513 "movq %4, %%mm6 \n\t"
514 "movq %5, %%mm7 \n\t"
517 PREFETCH" 32(%1) \n\t"
518 "movd (%1), %%mm0 \n\t"
519 "movd 4(%1), %%mm3 \n\t"
520 "punpckldq 8(%1), %%mm0 \n\t"
521 "punpckldq 12(%1), %%mm3 \n\t"
522 "movq %%mm0, %%mm1 \n\t"
523 "movq %%mm3, %%mm4 \n\t"
524 "pand %%mm6, %%mm0 \n\t"
525 "pand %%mm6, %%mm3 \n\t"
526 "pmaddwd %%mm7, %%mm0 \n\t"
527 "pmaddwd %%mm7, %%mm3 \n\t"
528 "pand %%mm5, %%mm1 \n\t"
529 "pand %%mm5, %%mm4 \n\t"
530 "por %%mm1, %%mm0 \n\t"
531 "por %%mm4, %%mm3 \n\t"
532 "psrld $6, %%mm0 \n\t"
533 "pslld $10, %%mm3 \n\t"
534 "por %%mm3, %%mm0 \n\t"
535 MOVNTQ" %%mm0, (%0) \n\t"
541 : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
544 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
548 ::"m"(red_15mask),"m"(green_15mask));
554 "movd 4%1, %%mm3\n\t"
555 "punpckldq 8%1, %%mm0\n\t"
556 "punpckldq 12%1, %%mm3\n\t"
557 "movq %%mm0, %%mm1\n\t"
558 "movq %%mm0, %%mm2\n\t"
559 "movq %%mm3, %%mm4\n\t"
560 "movq %%mm3, %%mm5\n\t"
561 "psrlq $3, %%mm0\n\t"
562 "psrlq $3, %%mm3\n\t"
565 "psrlq $6, %%mm1\n\t"
566 "psrlq $6, %%mm4\n\t"
567 "pand %%mm6, %%mm1\n\t"
568 "pand %%mm6, %%mm4\n\t"
569 "psrlq $9, %%mm2\n\t"
570 "psrlq $9, %%mm5\n\t"
571 "pand %%mm7, %%mm2\n\t"
572 "pand %%mm7, %%mm5\n\t"
573 "por %%mm1, %%mm0\n\t"
574 "por %%mm4, %%mm3\n\t"
575 "por %%mm2, %%mm0\n\t"
576 "por %%mm5, %%mm3\n\t"
577 "psllq $16, %%mm3\n\t"
578 "por %%mm3, %%mm0\n\t"
579 MOVNTQ" %%mm0, %0\n\t"
580 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
585 __asm __volatile(SFENCE:::"memory");
586 __asm __volatile(EMMS:::"memory");
590 register int rgb = *(uint32_t*)s; s += 4;
591 *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
595 static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
597 const uint8_t *s = src;
600 const uint8_t *mm_end;
602 uint16_t *d = (uint16_t *)dst;
605 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
609 ::"m"(red_15mask),"m"(green_15mask));
616 "movd 4%1, %%mm3\n\t"
617 "punpckldq 8%1, %%mm0\n\t"
618 "punpckldq 12%1, %%mm3\n\t"
619 "movq %%mm0, %%mm1\n\t"
620 "movq %%mm0, %%mm2\n\t"
621 "movq %%mm3, %%mm4\n\t"
622 "movq %%mm3, %%mm5\n\t"
623 "psllq $7, %%mm0\n\t"
624 "psllq $7, %%mm3\n\t"
625 "pand %%mm7, %%mm0\n\t"
626 "pand %%mm7, %%mm3\n\t"
627 "psrlq $6, %%mm1\n\t"
628 "psrlq $6, %%mm4\n\t"
629 "pand %%mm6, %%mm1\n\t"
630 "pand %%mm6, %%mm4\n\t"
631 "psrlq $19, %%mm2\n\t"
632 "psrlq $19, %%mm5\n\t"
635 "por %%mm1, %%mm0\n\t"
636 "por %%mm4, %%mm3\n\t"
637 "por %%mm2, %%mm0\n\t"
638 "por %%mm5, %%mm3\n\t"
639 "psllq $16, %%mm3\n\t"
640 "por %%mm3, %%mm0\n\t"
641 MOVNTQ" %%mm0, %0\n\t"
642 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
646 __asm __volatile(SFENCE:::"memory");
647 __asm __volatile(EMMS:::"memory");
651 register int rgb = *(uint32_t*)s; s += 4;
652 *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
656 static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, long src_size)
658 const uint8_t *s = src;
661 const uint8_t *mm_end;
663 uint16_t *d = (uint16_t *)dst;
666 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
670 ::"m"(red_16mask),"m"(green_16mask));
677 "movd 3%1, %%mm3\n\t"
678 "punpckldq 6%1, %%mm0\n\t"
679 "punpckldq 9%1, %%mm3\n\t"
680 "movq %%mm0, %%mm1\n\t"
681 "movq %%mm0, %%mm2\n\t"
682 "movq %%mm3, %%mm4\n\t"
683 "movq %%mm3, %%mm5\n\t"
684 "psrlq $3, %%mm0\n\t"
685 "psrlq $3, %%mm3\n\t"
688 "psrlq $5, %%mm1\n\t"
689 "psrlq $5, %%mm4\n\t"
690 "pand %%mm6, %%mm1\n\t"
691 "pand %%mm6, %%mm4\n\t"
692 "psrlq $8, %%mm2\n\t"
693 "psrlq $8, %%mm5\n\t"
694 "pand %%mm7, %%mm2\n\t"
695 "pand %%mm7, %%mm5\n\t"
696 "por %%mm1, %%mm0\n\t"
697 "por %%mm4, %%mm3\n\t"
698 "por %%mm2, %%mm0\n\t"
699 "por %%mm5, %%mm3\n\t"
700 "psllq $16, %%mm3\n\t"
701 "por %%mm3, %%mm0\n\t"
702 MOVNTQ" %%mm0, %0\n\t"
703 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
707 __asm __volatile(SFENCE:::"memory");
708 __asm __volatile(EMMS:::"memory");
715 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
719 static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, long src_size)
721 const uint8_t *s = src;
724 const uint8_t *mm_end;
726 uint16_t *d = (uint16_t *)dst;
729 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
733 ::"m"(red_16mask),"m"(green_16mask));
740 "movd 3%1, %%mm3\n\t"
741 "punpckldq 6%1, %%mm0\n\t"
742 "punpckldq 9%1, %%mm3\n\t"
743 "movq %%mm0, %%mm1\n\t"
744 "movq %%mm0, %%mm2\n\t"
745 "movq %%mm3, %%mm4\n\t"
746 "movq %%mm3, %%mm5\n\t"
747 "psllq $8, %%mm0\n\t"
748 "psllq $8, %%mm3\n\t"
749 "pand %%mm7, %%mm0\n\t"
750 "pand %%mm7, %%mm3\n\t"
751 "psrlq $5, %%mm1\n\t"
752 "psrlq $5, %%mm4\n\t"
753 "pand %%mm6, %%mm1\n\t"
754 "pand %%mm6, %%mm4\n\t"
755 "psrlq $19, %%mm2\n\t"
756 "psrlq $19, %%mm5\n\t"
759 "por %%mm1, %%mm0\n\t"
760 "por %%mm4, %%mm3\n\t"
761 "por %%mm2, %%mm0\n\t"
762 "por %%mm5, %%mm3\n\t"
763 "psllq $16, %%mm3\n\t"
764 "por %%mm3, %%mm0\n\t"
765 MOVNTQ" %%mm0, %0\n\t"
766 :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
770 __asm __volatile(SFENCE:::"memory");
771 __asm __volatile(EMMS:::"memory");
778 *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
782 static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, long src_size)
784 const uint8_t *s = src;
787 const uint8_t *mm_end;
789 uint16_t *d = (uint16_t *)dst;
792 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
796 ::"m"(red_15mask),"m"(green_15mask));
803 "movd 3%1, %%mm3\n\t"
804 "punpckldq 6%1, %%mm0\n\t"
805 "punpckldq 9%1, %%mm3\n\t"
806 "movq %%mm0, %%mm1\n\t"
807 "movq %%mm0, %%mm2\n\t"
808 "movq %%mm3, %%mm4\n\t"
809 "movq %%mm3, %%mm5\n\t"
810 "psrlq $3, %%mm0\n\t"
811 "psrlq $3, %%mm3\n\t"
814 "psrlq $6, %%mm1\n\t"
815 "psrlq $6, %%mm4\n\t"
816 "pand %%mm6, %%mm1\n\t"
817 "pand %%mm6, %%mm4\n\t"
818 "psrlq $9, %%mm2\n\t"
819 "psrlq $9, %%mm5\n\t"
820 "pand %%mm7, %%mm2\n\t"
821 "pand %%mm7, %%mm5\n\t"
822 "por %%mm1, %%mm0\n\t"
823 "por %%mm4, %%mm3\n\t"
824 "por %%mm2, %%mm0\n\t"
825 "por %%mm5, %%mm3\n\t"
826 "psllq $16, %%mm3\n\t"
827 "por %%mm3, %%mm0\n\t"
828 MOVNTQ" %%mm0, %0\n\t"
829 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
833 __asm __volatile(SFENCE:::"memory");
834 __asm __volatile(EMMS:::"memory");
841 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
845 static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, long src_size)
847 const uint8_t *s = src;
850 const uint8_t *mm_end;
852 uint16_t *d = (uint16_t *)dst;
855 __asm __volatile(PREFETCH" %0"::"m"(*src):"memory");
859 ::"m"(red_15mask),"m"(green_15mask));
866 "movd 3%1, %%mm3\n\t"
867 "punpckldq 6%1, %%mm0\n\t"
868 "punpckldq 9%1, %%mm3\n\t"
869 "movq %%mm0, %%mm1\n\t"
870 "movq %%mm0, %%mm2\n\t"
871 "movq %%mm3, %%mm4\n\t"
872 "movq %%mm3, %%mm5\n\t"
873 "psllq $7, %%mm0\n\t"
874 "psllq $7, %%mm3\n\t"
875 "pand %%mm7, %%mm0\n\t"
876 "pand %%mm7, %%mm3\n\t"
877 "psrlq $6, %%mm1\n\t"
878 "psrlq $6, %%mm4\n\t"
879 "pand %%mm6, %%mm1\n\t"
880 "pand %%mm6, %%mm4\n\t"
881 "psrlq $19, %%mm2\n\t"
882 "psrlq $19, %%mm5\n\t"
885 "por %%mm1, %%mm0\n\t"
886 "por %%mm4, %%mm3\n\t"
887 "por %%mm2, %%mm0\n\t"
888 "por %%mm5, %%mm3\n\t"
889 "psllq $16, %%mm3\n\t"
890 "por %%mm3, %%mm0\n\t"
891 MOVNTQ" %%mm0, %0\n\t"
892 :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
896 __asm __volatile(SFENCE:::"memory");
897 __asm __volatile(EMMS:::"memory");
904 *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
909 I use here less accurate approximation by simply
910 left-shifting the input
911 value and filling the low order bits with
912 zeroes. This method improves png's
913 compression but this scheme cannot reproduce white exactly, since it does not
914 generate an all-ones maximum value; the net effect is to darken the
917 The better method should be "left bit replication":
927 | Leftmost Bits Repeated to Fill Open Bits
931 static inline void RENAME(rgb15to24)(const uint8_t *src, uint8_t *dst, long src_size)
935 const uint16_t *mm_end;
937 uint8_t *d = (uint8_t *)dst;
938 const uint16_t *s = (uint16_t *)src;
939 end = s + src_size/2;
941 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
953 "psllq $3, %%mm0\n\t"
954 "psrlq $2, %%mm1\n\t"
955 "psrlq $7, %%mm2\n\t"
956 "movq %%mm0, %%mm3\n\t"
957 "movq %%mm1, %%mm4\n\t"
958 "movq %%mm2, %%mm5\n\t"
959 "punpcklwd %5, %%mm0\n\t"
960 "punpcklwd %5, %%mm1\n\t"
961 "punpcklwd %5, %%mm2\n\t"
962 "punpckhwd %5, %%mm3\n\t"
963 "punpckhwd %5, %%mm4\n\t"
964 "punpckhwd %5, %%mm5\n\t"
965 "psllq $8, %%mm1\n\t"
966 "psllq $16, %%mm2\n\t"
967 "por %%mm1, %%mm0\n\t"
968 "por %%mm2, %%mm0\n\t"
969 "psllq $8, %%mm4\n\t"
970 "psllq $16, %%mm5\n\t"
971 "por %%mm4, %%mm3\n\t"
972 "por %%mm5, %%mm3\n\t"
974 "movq %%mm0, %%mm6\n\t"
975 "movq %%mm3, %%mm7\n\t"
977 "movq 8%1, %%mm0\n\t"
978 "movq 8%1, %%mm1\n\t"
979 "movq 8%1, %%mm2\n\t"
983 "psllq $3, %%mm0\n\t"
984 "psrlq $2, %%mm1\n\t"
985 "psrlq $7, %%mm2\n\t"
986 "movq %%mm0, %%mm3\n\t"
987 "movq %%mm1, %%mm4\n\t"
988 "movq %%mm2, %%mm5\n\t"
989 "punpcklwd %5, %%mm0\n\t"
990 "punpcklwd %5, %%mm1\n\t"
991 "punpcklwd %5, %%mm2\n\t"
992 "punpckhwd %5, %%mm3\n\t"
993 "punpckhwd %5, %%mm4\n\t"
994 "punpckhwd %5, %%mm5\n\t"
995 "psllq $8, %%mm1\n\t"
996 "psllq $16, %%mm2\n\t"
997 "por %%mm1, %%mm0\n\t"
998 "por %%mm2, %%mm0\n\t"
999 "psllq $8, %%mm4\n\t"
1000 "psllq $16, %%mm5\n\t"
1001 "por %%mm4, %%mm3\n\t"
1002 "por %%mm5, %%mm3\n\t"
1005 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r), "m"(mmx_null)
1007 /* Borrowed 32 to 24 */
1009 "movq %%mm0, %%mm4\n\t"
1010 "movq %%mm3, %%mm5\n\t"
1011 "movq %%mm6, %%mm0\n\t"
1012 "movq %%mm7, %%mm1\n\t"
1014 "movq %%mm4, %%mm6\n\t"
1015 "movq %%mm5, %%mm7\n\t"
1016 "movq %%mm0, %%mm2\n\t"
1017 "movq %%mm1, %%mm3\n\t"
1019 "psrlq $8, %%mm2\n\t"
1020 "psrlq $8, %%mm3\n\t"
1021 "psrlq $8, %%mm6\n\t"
1022 "psrlq $8, %%mm7\n\t"
1023 "pand %2, %%mm0\n\t"
1024 "pand %2, %%mm1\n\t"
1025 "pand %2, %%mm4\n\t"
1026 "pand %2, %%mm5\n\t"
1027 "pand %3, %%mm2\n\t"
1028 "pand %3, %%mm3\n\t"
1029 "pand %3, %%mm6\n\t"
1030 "pand %3, %%mm7\n\t"
1031 "por %%mm2, %%mm0\n\t"
1032 "por %%mm3, %%mm1\n\t"
1033 "por %%mm6, %%mm4\n\t"
1034 "por %%mm7, %%mm5\n\t"
1036 "movq %%mm1, %%mm2\n\t"
1037 "movq %%mm4, %%mm3\n\t"
1038 "psllq $48, %%mm2\n\t"
1039 "psllq $32, %%mm3\n\t"
1040 "pand %4, %%mm2\n\t"
1041 "pand %5, %%mm3\n\t"
1042 "por %%mm2, %%mm0\n\t"
1043 "psrlq $16, %%mm1\n\t"
1044 "psrlq $32, %%mm4\n\t"
1045 "psllq $16, %%mm5\n\t"
1046 "por %%mm3, %%mm1\n\t"
1047 "pand %6, %%mm5\n\t"
1048 "por %%mm5, %%mm4\n\t"
1050 MOVNTQ" %%mm0, %0\n\t"
1051 MOVNTQ" %%mm1, 8%0\n\t"
1052 MOVNTQ" %%mm4, 16%0"
1055 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1060 __asm __volatile(SFENCE:::"memory");
1061 __asm __volatile(EMMS:::"memory");
1065 register uint16_t bgr;
1067 *d++ = (bgr&0x1F)<<3;
1068 *d++ = (bgr&0x3E0)>>2;
1069 *d++ = (bgr&0x7C00)>>7;
1073 static inline void RENAME(rgb16to24)(const uint8_t *src, uint8_t *dst, long src_size)
1075 const uint16_t *end;
1077 const uint16_t *mm_end;
1079 uint8_t *d = (uint8_t *)dst;
1080 const uint16_t *s = (const uint16_t *)src;
1081 end = s + src_size/2;
1083 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
1089 "movq %1, %%mm0\n\t"
1090 "movq %1, %%mm1\n\t"
1091 "movq %1, %%mm2\n\t"
1092 "pand %2, %%mm0\n\t"
1093 "pand %3, %%mm1\n\t"
1094 "pand %4, %%mm2\n\t"
1095 "psllq $3, %%mm0\n\t"
1096 "psrlq $3, %%mm1\n\t"
1097 "psrlq $8, %%mm2\n\t"
1098 "movq %%mm0, %%mm3\n\t"
1099 "movq %%mm1, %%mm4\n\t"
1100 "movq %%mm2, %%mm5\n\t"
1101 "punpcklwd %5, %%mm0\n\t"
1102 "punpcklwd %5, %%mm1\n\t"
1103 "punpcklwd %5, %%mm2\n\t"
1104 "punpckhwd %5, %%mm3\n\t"
1105 "punpckhwd %5, %%mm4\n\t"
1106 "punpckhwd %5, %%mm5\n\t"
1107 "psllq $8, %%mm1\n\t"
1108 "psllq $16, %%mm2\n\t"
1109 "por %%mm1, %%mm0\n\t"
1110 "por %%mm2, %%mm0\n\t"
1111 "psllq $8, %%mm4\n\t"
1112 "psllq $16, %%mm5\n\t"
1113 "por %%mm4, %%mm3\n\t"
1114 "por %%mm5, %%mm3\n\t"
1116 "movq %%mm0, %%mm6\n\t"
1117 "movq %%mm3, %%mm7\n\t"
1119 "movq 8%1, %%mm0\n\t"
1120 "movq 8%1, %%mm1\n\t"
1121 "movq 8%1, %%mm2\n\t"
1122 "pand %2, %%mm0\n\t"
1123 "pand %3, %%mm1\n\t"
1124 "pand %4, %%mm2\n\t"
1125 "psllq $3, %%mm0\n\t"
1126 "psrlq $3, %%mm1\n\t"
1127 "psrlq $8, %%mm2\n\t"
1128 "movq %%mm0, %%mm3\n\t"
1129 "movq %%mm1, %%mm4\n\t"
1130 "movq %%mm2, %%mm5\n\t"
1131 "punpcklwd %5, %%mm0\n\t"
1132 "punpcklwd %5, %%mm1\n\t"
1133 "punpcklwd %5, %%mm2\n\t"
1134 "punpckhwd %5, %%mm3\n\t"
1135 "punpckhwd %5, %%mm4\n\t"
1136 "punpckhwd %5, %%mm5\n\t"
1137 "psllq $8, %%mm1\n\t"
1138 "psllq $16, %%mm2\n\t"
1139 "por %%mm1, %%mm0\n\t"
1140 "por %%mm2, %%mm0\n\t"
1141 "psllq $8, %%mm4\n\t"
1142 "psllq $16, %%mm5\n\t"
1143 "por %%mm4, %%mm3\n\t"
1144 "por %%mm5, %%mm3\n\t"
1146 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null)
1148 /* Borrowed 32 to 24 */
1150 "movq %%mm0, %%mm4\n\t"
1151 "movq %%mm3, %%mm5\n\t"
1152 "movq %%mm6, %%mm0\n\t"
1153 "movq %%mm7, %%mm1\n\t"
1155 "movq %%mm4, %%mm6\n\t"
1156 "movq %%mm5, %%mm7\n\t"
1157 "movq %%mm0, %%mm2\n\t"
1158 "movq %%mm1, %%mm3\n\t"
1160 "psrlq $8, %%mm2\n\t"
1161 "psrlq $8, %%mm3\n\t"
1162 "psrlq $8, %%mm6\n\t"
1163 "psrlq $8, %%mm7\n\t"
1164 "pand %2, %%mm0\n\t"
1165 "pand %2, %%mm1\n\t"
1166 "pand %2, %%mm4\n\t"
1167 "pand %2, %%mm5\n\t"
1168 "pand %3, %%mm2\n\t"
1169 "pand %3, %%mm3\n\t"
1170 "pand %3, %%mm6\n\t"
1171 "pand %3, %%mm7\n\t"
1172 "por %%mm2, %%mm0\n\t"
1173 "por %%mm3, %%mm1\n\t"
1174 "por %%mm6, %%mm4\n\t"
1175 "por %%mm7, %%mm5\n\t"
1177 "movq %%mm1, %%mm2\n\t"
1178 "movq %%mm4, %%mm3\n\t"
1179 "psllq $48, %%mm2\n\t"
1180 "psllq $32, %%mm3\n\t"
1181 "pand %4, %%mm2\n\t"
1182 "pand %5, %%mm3\n\t"
1183 "por %%mm2, %%mm0\n\t"
1184 "psrlq $16, %%mm1\n\t"
1185 "psrlq $32, %%mm4\n\t"
1186 "psllq $16, %%mm5\n\t"
1187 "por %%mm3, %%mm1\n\t"
1188 "pand %6, %%mm5\n\t"
1189 "por %%mm5, %%mm4\n\t"
1191 MOVNTQ" %%mm0, %0\n\t"
1192 MOVNTQ" %%mm1, 8%0\n\t"
1193 MOVNTQ" %%mm4, 16%0"
1196 :"m"(*s),"m"(mask24l),"m"(mask24h),"m"(mask24hh),"m"(mask24hhh),"m"(mask24hhhh)
1201 __asm __volatile(SFENCE:::"memory");
1202 __asm __volatile(EMMS:::"memory");
1206 register uint16_t bgr;
1208 *d++ = (bgr&0x1F)<<3;
1209 *d++ = (bgr&0x7E0)>>3;
1210 *d++ = (bgr&0xF800)>>8;
1214 static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, long src_size)
1216 const uint16_t *end;
1218 const uint16_t *mm_end;
1220 uint8_t *d = (uint8_t *)dst;
1221 const uint16_t *s = (const uint16_t *)src;
1222 end = s + src_size/2;
1224 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
1225 __asm __volatile("pxor %%mm7,%%mm7\n\t":::"memory");
1231 "movq %1, %%mm0\n\t"
1232 "movq %1, %%mm1\n\t"
1233 "movq %1, %%mm2\n\t"
1234 "pand %2, %%mm0\n\t"
1235 "pand %3, %%mm1\n\t"
1236 "pand %4, %%mm2\n\t"
1237 "psllq $3, %%mm0\n\t"
1238 "psrlq $2, %%mm1\n\t"
1239 "psrlq $7, %%mm2\n\t"
1240 "movq %%mm0, %%mm3\n\t"
1241 "movq %%mm1, %%mm4\n\t"
1242 "movq %%mm2, %%mm5\n\t"
1243 "punpcklwd %%mm7, %%mm0\n\t"
1244 "punpcklwd %%mm7, %%mm1\n\t"
1245 "punpcklwd %%mm7, %%mm2\n\t"
1246 "punpckhwd %%mm7, %%mm3\n\t"
1247 "punpckhwd %%mm7, %%mm4\n\t"
1248 "punpckhwd %%mm7, %%mm5\n\t"
1249 "psllq $8, %%mm1\n\t"
1250 "psllq $16, %%mm2\n\t"
1251 "por %%mm1, %%mm0\n\t"
1252 "por %%mm2, %%mm0\n\t"
1253 "psllq $8, %%mm4\n\t"
1254 "psllq $16, %%mm5\n\t"
1255 "por %%mm4, %%mm3\n\t"
1256 "por %%mm5, %%mm3\n\t"
1257 MOVNTQ" %%mm0, %0\n\t"
1258 MOVNTQ" %%mm3, 8%0\n\t"
1260 :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r)
1265 __asm __volatile(SFENCE:::"memory");
1266 __asm __volatile(EMMS:::"memory");
1270 #if 0 //slightly slower on athlon
1272 *((uint32_t*)d)++ = ((bgr&0x1F)<<3) + ((bgr&0x3E0)<<6) + ((bgr&0x7C00)<<9);
1274 register uint16_t bgr;
1276 #ifdef WORDS_BIGENDIAN
1278 *d++ = (bgr&0x7C00)>>7;
1279 *d++ = (bgr&0x3E0)>>2;
1280 *d++ = (bgr&0x1F)<<3;
1282 *d++ = (bgr&0x1F)<<3;
1283 *d++ = (bgr&0x3E0)>>2;
1284 *d++ = (bgr&0x7C00)>>7;
1292 static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, long src_size)
1294 const uint16_t *end;
1296 const uint16_t *mm_end;
1298 uint8_t *d = (uint8_t *)dst;
1299 const uint16_t *s = (uint16_t *)src;
1300 end = s + src_size/2;
1302 __asm __volatile(PREFETCH" %0"::"m"(*s):"memory");
1303 __asm __volatile("pxor %%mm7,%%mm7\n\t":::"memory");
1309 "movq %1, %%mm0\n\t"
1310 "movq %1, %%mm1\n\t"
1311 "movq %1, %%mm2\n\t"
1312 "pand %2, %%mm0\n\t"
1313 "pand %3, %%mm1\n\t"
1314 "pand %4, %%mm2\n\t"
1315 "psllq $3, %%mm0\n\t"
1316 "psrlq $3, %%mm1\n\t"
1317 "psrlq $8, %%mm2\n\t"
1318 "movq %%mm0, %%mm3\n\t"
1319 "movq %%mm1, %%mm4\n\t"
1320 "movq %%mm2, %%mm5\n\t"
1321 "punpcklwd %%mm7, %%mm0\n\t"
1322 "punpcklwd %%mm7, %%mm1\n\t"
1323 "punpcklwd %%mm7, %%mm2\n\t"
1324 "punpckhwd %%mm7, %%mm3\n\t"
1325 "punpckhwd %%mm7, %%mm4\n\t"
1326 "punpckhwd %%mm7, %%mm5\n\t"
1327 "psllq $8, %%mm1\n\t"
1328 "psllq $16, %%mm2\n\t"
1329 "por %%mm1, %%mm0\n\t"
1330 "por %%mm2, %%mm0\n\t"
1331 "psllq $8, %%mm4\n\t"
1332 "psllq $16, %%mm5\n\t"
1333 "por %%mm4, %%mm3\n\t"
1334 "por %%mm5, %%mm3\n\t"
1335 MOVNTQ" %%mm0, %0\n\t"
1336 MOVNTQ" %%mm3, 8%0\n\t"
1338 :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r)
1343 __asm __volatile(SFENCE:::"memory");
1344 __asm __volatile(EMMS:::"memory");
1348 register uint16_t bgr;
1350 #ifdef WORDS_BIGENDIAN
1352 *d++ = (bgr&0xF800)>>8;
1353 *d++ = (bgr&0x7E0)>>3;
1354 *d++ = (bgr&0x1F)<<3;
1356 *d++ = (bgr&0x1F)<<3;
1357 *d++ = (bgr&0x7E0)>>3;
1358 *d++ = (bgr&0xF800)>>8;
1364 static inline void RENAME(rgb32tobgr32)(const uint8_t *src, uint8_t *dst, long src_size)
1367 /* TODO: unroll this loop */
1369 "xor %%"REG_a", %%"REG_a" \n\t"
1372 PREFETCH" 32(%0, %%"REG_a") \n\t"
1373 "movq (%0, %%"REG_a"), %%mm0 \n\t"
1374 "movq %%mm0, %%mm1 \n\t"
1375 "movq %%mm0, %%mm2 \n\t"
1376 "pslld $16, %%mm0 \n\t"
1377 "psrld $16, %%mm1 \n\t"
1378 "pand "MANGLE(mask32r)", %%mm0 \n\t"
1379 "pand "MANGLE(mask32g)", %%mm2 \n\t"
1380 "pand "MANGLE(mask32b)", %%mm1 \n\t"
1381 "por %%mm0, %%mm2 \n\t"
1382 "por %%mm1, %%mm2 \n\t"
1383 MOVNTQ" %%mm2, (%1, %%"REG_a") \n\t"
1384 "add $8, %%"REG_a" \n\t"
1385 "cmp %2, %%"REG_a" \n\t"
1387 :: "r" (src), "r"(dst), "r" (src_size-7)
1391 __asm __volatile(SFENCE:::"memory");
1392 __asm __volatile(EMMS:::"memory");
1395 unsigned num_pixels = src_size >> 2;
1396 for(i=0; i<num_pixels; i++)
1398 #ifdef WORDS_BIGENDIAN
1399 dst[4*i + 1] = src[4*i + 3];
1400 dst[4*i + 2] = src[4*i + 2];
1401 dst[4*i + 3] = src[4*i + 1];
1403 dst[4*i + 0] = src[4*i + 2];
1404 dst[4*i + 1] = src[4*i + 1];
1405 dst[4*i + 2] = src[4*i + 0];
1411 static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, long src_size)
1415 long mmx_size= 23 - src_size;
1417 "movq "MANGLE(mask24r)", %%mm5 \n\t"
1418 "movq "MANGLE(mask24g)", %%mm6 \n\t"
1419 "movq "MANGLE(mask24b)", %%mm7 \n\t"
1422 PREFETCH" 32(%1, %%"REG_a") \n\t"
1423 "movq (%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1424 "movq (%1, %%"REG_a"), %%mm1 \n\t" // BGR BGR BG
1425 "movq 2(%1, %%"REG_a"), %%mm2 \n\t" // R BGR BGR B
1426 "psllq $16, %%mm0 \n\t" // 00 BGR BGR
1427 "pand %%mm5, %%mm0 \n\t"
1428 "pand %%mm6, %%mm1 \n\t"
1429 "pand %%mm7, %%mm2 \n\t"
1430 "por %%mm0, %%mm1 \n\t"
1431 "por %%mm2, %%mm1 \n\t"
1432 "movq 6(%1, %%"REG_a"), %%mm0 \n\t" // BGR BGR BG
1433 MOVNTQ" %%mm1, (%2, %%"REG_a")\n\t" // RGB RGB RG
1434 "movq 8(%1, %%"REG_a"), %%mm1 \n\t" // R BGR BGR B
1435 "movq 10(%1, %%"REG_a"), %%mm2 \n\t" // GR BGR BGR
1436 "pand %%mm7, %%mm0 \n\t"
1437 "pand %%mm5, %%mm1 \n\t"
1438 "pand %%mm6, %%mm2 \n\t"
1439 "por %%mm0, %%mm1 \n\t"
1440 "por %%mm2, %%mm1 \n\t"
1441 "movq 14(%1, %%"REG_a"), %%mm0 \n\t" // R BGR BGR B
1442 MOVNTQ" %%mm1, 8(%2, %%"REG_a")\n\t" // B RGB RGB R
1443 "movq 16(%1, %%"REG_a"), %%mm1 \n\t" // GR BGR BGR
1444 "movq 18(%1, %%"REG_a"), %%mm2 \n\t" // BGR BGR BG
1445 "pand %%mm6, %%mm0 \n\t"
1446 "pand %%mm7, %%mm1 \n\t"
1447 "pand %%mm5, %%mm2 \n\t"
1448 "por %%mm0, %%mm1 \n\t"
1449 "por %%mm2, %%mm1 \n\t"
1450 MOVNTQ" %%mm1, 16(%2, %%"REG_a")\n\t"
1451 "add $24, %%"REG_a" \n\t"
1454 : "r" (src-mmx_size), "r"(dst-mmx_size)
1457 __asm __volatile(SFENCE:::"memory");
1458 __asm __volatile(EMMS:::"memory");
1460 if(mmx_size==23) return; //finihsed, was multiple of 8
1464 src_size= 23-mmx_size;
1468 for(i=0; i<src_size; i+=3)
1472 dst[i + 1] = src[i + 1];
1473 dst[i + 2] = src[i + 0];
1478 static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1479 long width, long height,
1480 long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1483 const long chromWidth= width>>1;
1484 for(y=0; y<height; y++)
1487 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1489 "xor %%"REG_a", %%"REG_a" \n\t"
1492 PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1493 PREFETCH" 32(%2, %%"REG_a") \n\t"
1494 PREFETCH" 32(%3, %%"REG_a") \n\t"
1495 "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1496 "movq %%mm0, %%mm2 \n\t" // U(0)
1497 "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1498 "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1499 "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1501 "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1502 "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1503 "movq %%mm3, %%mm4 \n\t" // Y(0)
1504 "movq %%mm5, %%mm6 \n\t" // Y(8)
1505 "punpcklbw %%mm0, %%mm3 \n\t" // YUYV YUYV(0)
1506 "punpckhbw %%mm0, %%mm4 \n\t" // YUYV YUYV(4)
1507 "punpcklbw %%mm2, %%mm5 \n\t" // YUYV YUYV(8)
1508 "punpckhbw %%mm2, %%mm6 \n\t" // YUYV YUYV(12)
1510 MOVNTQ" %%mm3, (%0, %%"REG_a", 4)\n\t"
1511 MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1512 MOVNTQ" %%mm5, 16(%0, %%"REG_a", 4)\n\t"
1513 MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1515 "add $8, %%"REG_a" \n\t"
1516 "cmp %4, %%"REG_a" \n\t"
1518 ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1523 #if defined ARCH_ALPHA && defined HAVE_MVI
1524 #define pl2yuy2(n) \
1529 asm("unpkbw %1, %0" : "=r"(y1) : "r"(y1)); \
1530 asm("unpkbw %1, %0" : "=r"(y2) : "r"(y2)); \
1531 asm("unpkbl %1, %0" : "=r"(u) : "r"(u)); \
1532 asm("unpkbl %1, %0" : "=r"(v) : "r"(v)); \
1533 yuv1 = (u << 8) + (v << 24); \
1540 uint64_t *qdst = (uint64_t *) dst;
1541 uint64_t *qdst2 = (uint64_t *) (dst + dstStride);
1542 const uint32_t *yc = (uint32_t *) ysrc;
1543 const uint32_t *yc2 = (uint32_t *) (ysrc + lumStride);
1544 const uint16_t *uc = (uint16_t*) usrc, *vc = (uint16_t*) vsrc;
1545 for(i = 0; i < chromWidth; i += 8){
1546 uint64_t y1, y2, yuv1, yuv2;
1549 asm("ldq $31,64(%0)" :: "r"(yc));
1550 asm("ldq $31,64(%0)" :: "r"(yc2));
1551 asm("ldq $31,64(%0)" :: "r"(uc));
1552 asm("ldq $31,64(%0)" :: "r"(vc));
1570 #elif __WORDSIZE >= 64
1572 uint64_t *ldst = (uint64_t *) dst;
1573 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1574 for(i = 0; i < chromWidth; i += 2){
1576 k = yc[0] + (uc[0] << 8) +
1577 (yc[1] << 16) + (vc[0] << 24);
1578 l = yc[2] + (uc[1] << 8) +
1579 (yc[3] << 16) + (vc[1] << 24);
1580 *ldst++ = k + (l << 32);
1587 int i, *idst = (int32_t *) dst;
1588 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1589 for(i = 0; i < chromWidth; i++){
1590 #ifdef WORDS_BIGENDIAN
1591 *idst++ = (yc[0] << 24)+ (uc[0] << 16) +
1592 (yc[1] << 8) + (vc[0] << 0);
1594 *idst++ = yc[0] + (uc[0] << 8) +
1595 (yc[1] << 16) + (vc[0] << 24);
1603 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1605 usrc += chromStride;
1606 vsrc += chromStride;
1620 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1621 * problem for anyone then tell me, and ill fix it)
1623 static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1624 long width, long height,
1625 long lumStride, long chromStride, long dstStride)
1627 //FIXME interpolate chroma
1628 RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1631 static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1632 long width, long height,
1633 long lumStride, long chromStride, long dstStride, long vertLumPerChroma)
1636 const long chromWidth= width>>1;
1637 for(y=0; y<height; y++)
1640 //FIXME handle 2 lines a once (fewer prefetch, reuse some chrom, but very likely limited by mem anyway)
1642 "xor %%"REG_a", %%"REG_a" \n\t"
1645 PREFETCH" 32(%1, %%"REG_a", 2) \n\t"
1646 PREFETCH" 32(%2, %%"REG_a") \n\t"
1647 PREFETCH" 32(%3, %%"REG_a") \n\t"
1648 "movq (%2, %%"REG_a"), %%mm0 \n\t" // U(0)
1649 "movq %%mm0, %%mm2 \n\t" // U(0)
1650 "movq (%3, %%"REG_a"), %%mm1 \n\t" // V(0)
1651 "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1652 "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
1654 "movq (%1, %%"REG_a",2), %%mm3 \n\t" // Y(0)
1655 "movq 8(%1, %%"REG_a",2), %%mm5 \n\t" // Y(8)
1656 "movq %%mm0, %%mm4 \n\t" // Y(0)
1657 "movq %%mm2, %%mm6 \n\t" // Y(8)
1658 "punpcklbw %%mm3, %%mm0 \n\t" // YUYV YUYV(0)
1659 "punpckhbw %%mm3, %%mm4 \n\t" // YUYV YUYV(4)
1660 "punpcklbw %%mm5, %%mm2 \n\t" // YUYV YUYV(8)
1661 "punpckhbw %%mm5, %%mm6 \n\t" // YUYV YUYV(12)
1663 MOVNTQ" %%mm0, (%0, %%"REG_a", 4)\n\t"
1664 MOVNTQ" %%mm4, 8(%0, %%"REG_a", 4)\n\t"
1665 MOVNTQ" %%mm2, 16(%0, %%"REG_a", 4)\n\t"
1666 MOVNTQ" %%mm6, 24(%0, %%"REG_a", 4)\n\t"
1668 "add $8, %%"REG_a" \n\t"
1669 "cmp %4, %%"REG_a" \n\t"
1671 ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
1675 //FIXME adapt the alpha asm code from yv12->yuy2
1677 #if __WORDSIZE >= 64
1679 uint64_t *ldst = (uint64_t *) dst;
1680 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1681 for(i = 0; i < chromWidth; i += 2){
1683 k = uc[0] + (yc[0] << 8) +
1684 (vc[0] << 16) + (yc[1] << 24);
1685 l = uc[1] + (yc[2] << 8) +
1686 (vc[1] << 16) + (yc[3] << 24);
1687 *ldst++ = k + (l << 32);
1694 int i, *idst = (int32_t *) dst;
1695 const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
1696 for(i = 0; i < chromWidth; i++){
1697 #ifdef WORDS_BIGENDIAN
1698 *idst++ = (uc[0] << 24)+ (yc[0] << 16) +
1699 (vc[0] << 8) + (yc[1] << 0);
1701 *idst++ = uc[0] + (yc[0] << 8) +
1702 (vc[0] << 16) + (yc[1] << 24);
1710 if((y&(vertLumPerChroma-1))==(vertLumPerChroma-1) )
1712 usrc += chromStride;
1713 vsrc += chromStride;
1727 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1728 * problem for anyone then tell me, and ill fix it)
1730 static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1731 long width, long height,
1732 long lumStride, long chromStride, long dstStride)
1734 //FIXME interpolate chroma
1735 RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
1740 * width should be a multiple of 16
1742 static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
1743 long width, long height,
1744 long lumStride, long chromStride, long dstStride)
1746 RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
1751 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1752 * problem for anyone then tell me, and ill fix it)
1754 static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1755 long width, long height,
1756 long lumStride, long chromStride, long srcStride)
1759 const long chromWidth= width>>1;
1760 for(y=0; y<height; y+=2)
1764 "xor %%"REG_a", %%"REG_a" \n\t"
1765 "pcmpeqw %%mm7, %%mm7 \n\t"
1766 "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
1769 PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1770 "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1771 "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1772 "movq %%mm0, %%mm2 \n\t" // YUYV YUYV(0)
1773 "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(4)
1774 "psrlw $8, %%mm0 \n\t" // U0V0 U0V0(0)
1775 "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(4)
1776 "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
1777 "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
1778 "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
1779 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
1781 MOVNTQ" %%mm2, (%1, %%"REG_a", 2)\n\t"
1783 "movq 16(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(8)
1784 "movq 24(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(12)
1785 "movq %%mm1, %%mm3 \n\t" // YUYV YUYV(8)
1786 "movq %%mm2, %%mm4 \n\t" // YUYV YUYV(12)
1787 "psrlw $8, %%mm1 \n\t" // U0V0 U0V0(8)
1788 "psrlw $8, %%mm2 \n\t" // U0V0 U0V0(12)
1789 "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
1790 "pand %%mm7, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
1791 "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
1792 "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
1794 MOVNTQ" %%mm3, 8(%1, %%"REG_a", 2)\n\t"
1796 "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
1797 "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
1798 "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
1799 "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
1800 "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
1801 "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
1802 "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
1803 "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
1805 MOVNTQ" %%mm0, (%3, %%"REG_a") \n\t"
1806 MOVNTQ" %%mm2, (%2, %%"REG_a") \n\t"
1808 "add $8, %%"REG_a" \n\t"
1809 "cmp %4, %%"REG_a" \n\t"
1811 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1812 : "memory", "%"REG_a
1819 "xor %%"REG_a", %%"REG_a" \n\t"
1822 PREFETCH" 64(%0, %%"REG_a", 4) \n\t"
1823 "movq (%0, %%"REG_a", 4), %%mm0 \n\t" // YUYV YUYV(0)
1824 "movq 8(%0, %%"REG_a", 4), %%mm1\n\t" // YUYV YUYV(4)
1825 "movq 16(%0, %%"REG_a", 4), %%mm2\n\t" // YUYV YUYV(8)
1826 "movq 24(%0, %%"REG_a", 4), %%mm3\n\t" // YUYV YUYV(12)
1827 "pand %%mm7, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
1828 "pand %%mm7, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
1829 "pand %%mm7, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
1830 "pand %%mm7, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
1831 "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
1832 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
1834 MOVNTQ" %%mm0, (%1, %%"REG_a", 2)\n\t"
1835 MOVNTQ" %%mm2, 8(%1, %%"REG_a", 2)\n\t"
1837 "add $8, %%"REG_a" \n\t"
1838 "cmp %4, %%"REG_a" \n\t"
1841 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
1842 : "memory", "%"REG_a
1846 for(i=0; i<chromWidth; i++)
1848 ydst[2*i+0] = src[4*i+0];
1849 udst[i] = src[4*i+1];
1850 ydst[2*i+1] = src[4*i+2];
1851 vdst[i] = src[4*i+3];
1856 for(i=0; i<chromWidth; i++)
1858 ydst[2*i+0] = src[4*i+0];
1859 ydst[2*i+1] = src[4*i+2];
1862 udst += chromStride;
1863 vdst += chromStride;
1868 asm volatile( EMMS" \n\t"
1874 static inline void RENAME(yvu9toyv12)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,
1875 uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1876 long width, long height, long lumStride, long chromStride)
1879 memcpy(ydst, ysrc, width*height);
1881 /* XXX: implement upscaling for U,V */
1884 static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, long srcWidth, long srcHeight, long srcStride, long dstStride)
1891 for(x=0; x<srcWidth-1; x++){
1892 dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1893 dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1895 dst[2*srcWidth-1]= src[srcWidth-1];
1899 for(y=1; y<srcHeight; y++){
1900 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
1901 const long mmxSize= srcWidth&~15;
1903 "mov %4, %%"REG_a" \n\t"
1905 "movq (%0, %%"REG_a"), %%mm0 \n\t"
1906 "movq (%1, %%"REG_a"), %%mm1 \n\t"
1907 "movq 1(%0, %%"REG_a"), %%mm2 \n\t"
1908 "movq 1(%1, %%"REG_a"), %%mm3 \n\t"
1909 "movq -1(%0, %%"REG_a"), %%mm4 \n\t"
1910 "movq -1(%1, %%"REG_a"), %%mm5 \n\t"
1911 PAVGB" %%mm0, %%mm5 \n\t"
1912 PAVGB" %%mm0, %%mm3 \n\t"
1913 PAVGB" %%mm0, %%mm5 \n\t"
1914 PAVGB" %%mm0, %%mm3 \n\t"
1915 PAVGB" %%mm1, %%mm4 \n\t"
1916 PAVGB" %%mm1, %%mm2 \n\t"
1917 PAVGB" %%mm1, %%mm4 \n\t"
1918 PAVGB" %%mm1, %%mm2 \n\t"
1919 "movq %%mm5, %%mm7 \n\t"
1920 "movq %%mm4, %%mm6 \n\t"
1921 "punpcklbw %%mm3, %%mm5 \n\t"
1922 "punpckhbw %%mm3, %%mm7 \n\t"
1923 "punpcklbw %%mm2, %%mm4 \n\t"
1924 "punpckhbw %%mm2, %%mm6 \n\t"
1926 MOVNTQ" %%mm5, (%2, %%"REG_a", 2)\n\t"
1927 MOVNTQ" %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1928 MOVNTQ" %%mm4, (%3, %%"REG_a", 2)\n\t"
1929 MOVNTQ" %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1931 "movq %%mm5, (%2, %%"REG_a", 2) \n\t"
1932 "movq %%mm7, 8(%2, %%"REG_a", 2)\n\t"
1933 "movq %%mm4, (%3, %%"REG_a", 2) \n\t"
1934 "movq %%mm6, 8(%3, %%"REG_a", 2)\n\t"
1936 "add $8, %%"REG_a" \n\t"
1938 :: "r" (src + mmxSize ), "r" (src + srcStride + mmxSize ),
1939 "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
1945 const long mmxSize=1;
1947 dst[0 ]= (3*src[0] + src[srcStride])>>2;
1948 dst[dstStride]= ( src[0] + 3*src[srcStride])>>2;
1950 for(x=mmxSize-1; x<srcWidth-1; x++){
1951 dst[2*x +1]= (3*src[x+0] + src[x+srcStride+1])>>2;
1952 dst[2*x+dstStride+2]= ( src[x+0] + 3*src[x+srcStride+1])>>2;
1953 dst[2*x+dstStride+1]= ( src[x+1] + 3*src[x+srcStride ])>>2;
1954 dst[2*x +2]= (3*src[x+1] + src[x+srcStride ])>>2;
1956 dst[srcWidth*2 -1 ]= (3*src[srcWidth-1] + src[srcWidth-1 + srcStride])>>2;
1957 dst[srcWidth*2 -1 + dstStride]= ( src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
1967 for(x=0; x<srcWidth-1; x++){
1968 dst[2*x+1]= (3*src[x] + src[x+1])>>2;
1969 dst[2*x+2]= ( src[x] + 3*src[x+1])>>2;
1971 dst[2*srcWidth-1]= src[srcWidth-1];
1973 for(x=0; x<srcWidth; x++){
1980 asm volatile( EMMS" \n\t"
1988 * height should be a multiple of 2 and width should be a multiple of 16 (if this is a
1989 * problem for anyone then tell me, and ill fix it)
1990 * chrominance data is only taken from every secound line others are ignored FIXME write HQ version
1992 static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
1993 long width, long height,
1994 long lumStride, long chromStride, long srcStride)
1997 const long chromWidth= width>>1;
1998 for(y=0; y<height; y+=2)
2002 "xorl %%eax, %%eax \n\t"
2003 "pcmpeqw %%mm7, %%mm7 \n\t"
2004 "psrlw $8, %%mm7 \n\t" // FF,00,FF,00...
2007 PREFETCH" 64(%0, %%eax, 4) \n\t"
2008 "movq (%0, %%eax, 4), %%mm0 \n\t" // UYVY UYVY(0)
2009 "movq 8(%0, %%eax, 4), %%mm1 \n\t" // UYVY UYVY(4)
2010 "movq %%mm0, %%mm2 \n\t" // UYVY UYVY(0)
2011 "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(4)
2012 "pand %%mm7, %%mm0 \n\t" // U0V0 U0V0(0)
2013 "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(4)
2014 "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(0)
2015 "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(4)
2016 "packuswb %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
2017 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(0)
2019 MOVNTQ" %%mm2, (%1, %%eax, 2) \n\t"
2021 "movq 16(%0, %%eax, 4), %%mm1 \n\t" // UYVY UYVY(8)
2022 "movq 24(%0, %%eax, 4), %%mm2 \n\t" // UYVY UYVY(12)
2023 "movq %%mm1, %%mm3 \n\t" // UYVY UYVY(8)
2024 "movq %%mm2, %%mm4 \n\t" // UYVY UYVY(12)
2025 "pand %%mm7, %%mm1 \n\t" // U0V0 U0V0(8)
2026 "pand %%mm7, %%mm2 \n\t" // U0V0 U0V0(12)
2027 "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(8)
2028 "psrlw $8, %%mm4 \n\t" // Y0Y0 Y0Y0(12)
2029 "packuswb %%mm2, %%mm1 \n\t" // UVUV UVUV(8)
2030 "packuswb %%mm4, %%mm3 \n\t" // YYYY YYYY(8)
2032 MOVNTQ" %%mm3, 8(%1, %%eax, 2) \n\t"
2034 "movq %%mm0, %%mm2 \n\t" // UVUV UVUV(0)
2035 "movq %%mm1, %%mm3 \n\t" // UVUV UVUV(8)
2036 "psrlw $8, %%mm0 \n\t" // V0V0 V0V0(0)
2037 "psrlw $8, %%mm1 \n\t" // V0V0 V0V0(8)
2038 "pand %%mm7, %%mm2 \n\t" // U0U0 U0U0(0)
2039 "pand %%mm7, %%mm3 \n\t" // U0U0 U0U0(8)
2040 "packuswb %%mm1, %%mm0 \n\t" // VVVV VVVV(0)
2041 "packuswb %%mm3, %%mm2 \n\t" // UUUU UUUU(0)
2043 MOVNTQ" %%mm0, (%3, %%eax) \n\t"
2044 MOVNTQ" %%mm2, (%2, %%eax) \n\t"
2046 "addl $8, %%eax \n\t"
2047 "cmpl %4, %%eax \n\t"
2049 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2057 "xorl %%eax, %%eax \n\t"
2060 PREFETCH" 64(%0, %%eax, 4) \n\t"
2061 "movq (%0, %%eax, 4), %%mm0 \n\t" // YUYV YUYV(0)
2062 "movq 8(%0, %%eax, 4), %%mm1 \n\t" // YUYV YUYV(4)
2063 "movq 16(%0, %%eax, 4), %%mm2 \n\t" // YUYV YUYV(8)
2064 "movq 24(%0, %%eax, 4), %%mm3 \n\t" // YUYV YUYV(12)
2065 "psrlw $8, %%mm0 \n\t" // Y0Y0 Y0Y0(0)
2066 "psrlw $8, %%mm1 \n\t" // Y0Y0 Y0Y0(4)
2067 "psrlw $8, %%mm2 \n\t" // Y0Y0 Y0Y0(8)
2068 "psrlw $8, %%mm3 \n\t" // Y0Y0 Y0Y0(12)
2069 "packuswb %%mm1, %%mm0 \n\t" // YYYY YYYY(0)
2070 "packuswb %%mm3, %%mm2 \n\t" // YYYY YYYY(8)
2072 MOVNTQ" %%mm0, (%1, %%eax, 2) \n\t"
2073 MOVNTQ" %%mm2, 8(%1, %%eax, 2) \n\t"
2075 "addl $8, %%eax \n\t"
2076 "cmpl %4, %%eax \n\t"
2079 ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
2084 for(i=0; i<chromWidth; i++)
2086 udst[i] = src[4*i+0];
2087 ydst[2*i+0] = src[4*i+1];
2088 vdst[i] = src[4*i+2];
2089 ydst[2*i+1] = src[4*i+3];
2094 for(i=0; i<chromWidth; i++)
2096 ydst[2*i+0] = src[4*i+1];
2097 ydst[2*i+1] = src[4*i+3];
2100 udst += chromStride;
2101 vdst += chromStride;
2106 asm volatile( EMMS" \n\t"
2114 * height should be a multiple of 2 and width should be a multiple of 2 (if this is a
2115 * problem for anyone then tell me, and ill fix it)
2116 * chrominance data is only taken from every secound line others are ignored in the C version FIXME write HQ version
2118 static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
2119 long width, long height,
2120 long lumStride, long chromStride, long srcStride)
2123 const long chromWidth= width>>1;
2125 for(y=0; y<height-2; y+=2)
2131 "mov %2, %%"REG_a" \n\t"
2132 "movq "MANGLE(bgr2YCoeff)", %%mm6 \n\t"
2133 "movq "MANGLE(w1111)", %%mm5 \n\t"
2134 "pxor %%mm7, %%mm7 \n\t"
2135 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2138 PREFETCH" 64(%0, %%"REG_d") \n\t"
2139 "movd (%0, %%"REG_d"), %%mm0 \n\t"
2140 "movd 3(%0, %%"REG_d"), %%mm1 \n\t"
2141 "punpcklbw %%mm7, %%mm0 \n\t"
2142 "punpcklbw %%mm7, %%mm1 \n\t"
2143 "movd 6(%0, %%"REG_d"), %%mm2 \n\t"
2144 "movd 9(%0, %%"REG_d"), %%mm3 \n\t"
2145 "punpcklbw %%mm7, %%mm2 \n\t"
2146 "punpcklbw %%mm7, %%mm3 \n\t"
2147 "pmaddwd %%mm6, %%mm0 \n\t"
2148 "pmaddwd %%mm6, %%mm1 \n\t"
2149 "pmaddwd %%mm6, %%mm2 \n\t"
2150 "pmaddwd %%mm6, %%mm3 \n\t"
2151 #ifndef FAST_BGR2YV12
2152 "psrad $8, %%mm0 \n\t"
2153 "psrad $8, %%mm1 \n\t"
2154 "psrad $8, %%mm2 \n\t"
2155 "psrad $8, %%mm3 \n\t"
2157 "packssdw %%mm1, %%mm0 \n\t"
2158 "packssdw %%mm3, %%mm2 \n\t"
2159 "pmaddwd %%mm5, %%mm0 \n\t"
2160 "pmaddwd %%mm5, %%mm2 \n\t"
2161 "packssdw %%mm2, %%mm0 \n\t"
2162 "psraw $7, %%mm0 \n\t"
2164 "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
2165 "movd 15(%0, %%"REG_d"), %%mm1 \n\t"
2166 "punpcklbw %%mm7, %%mm4 \n\t"
2167 "punpcklbw %%mm7, %%mm1 \n\t"
2168 "movd 18(%0, %%"REG_d"), %%mm2 \n\t"
2169 "movd 21(%0, %%"REG_d"), %%mm3 \n\t"
2170 "punpcklbw %%mm7, %%mm2 \n\t"
2171 "punpcklbw %%mm7, %%mm3 \n\t"
2172 "pmaddwd %%mm6, %%mm4 \n\t"
2173 "pmaddwd %%mm6, %%mm1 \n\t"
2174 "pmaddwd %%mm6, %%mm2 \n\t"
2175 "pmaddwd %%mm6, %%mm3 \n\t"
2176 #ifndef FAST_BGR2YV12
2177 "psrad $8, %%mm4 \n\t"
2178 "psrad $8, %%mm1 \n\t"
2179 "psrad $8, %%mm2 \n\t"
2180 "psrad $8, %%mm3 \n\t"
2182 "packssdw %%mm1, %%mm4 \n\t"
2183 "packssdw %%mm3, %%mm2 \n\t"
2184 "pmaddwd %%mm5, %%mm4 \n\t"
2185 "pmaddwd %%mm5, %%mm2 \n\t"
2186 "add $24, %%"REG_d" \n\t"
2187 "packssdw %%mm2, %%mm4 \n\t"
2188 "psraw $7, %%mm4 \n\t"
2190 "packuswb %%mm4, %%mm0 \n\t"
2191 "paddusb "MANGLE(bgr2YOffset)", %%mm0 \n\t"
2193 MOVNTQ" %%mm0, (%1, %%"REG_a") \n\t"
2194 "add $8, %%"REG_a" \n\t"
2196 : : "r" (src+width*3), "r" (ydst+width), "g" (-width)
2197 : "%"REG_a, "%"REG_d
2204 "mov %4, %%"REG_a" \n\t"
2205 "movq "MANGLE(w1111)", %%mm5 \n\t"
2206 "movq "MANGLE(bgr2UCoeff)", %%mm6 \n\t"
2207 "pxor %%mm7, %%mm7 \n\t"
2208 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"\n\t"
2209 "add %%"REG_d", %%"REG_d" \n\t"
2212 PREFETCH" 64(%0, %%"REG_d") \n\t"
2213 PREFETCH" 64(%1, %%"REG_d") \n\t"
2214 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2215 "movq (%0, %%"REG_d"), %%mm0 \n\t"
2216 "movq (%1, %%"REG_d"), %%mm1 \n\t"
2217 "movq 6(%0, %%"REG_d"), %%mm2 \n\t"
2218 "movq 6(%1, %%"REG_d"), %%mm3 \n\t"
2219 PAVGB" %%mm1, %%mm0 \n\t"
2220 PAVGB" %%mm3, %%mm2 \n\t"
2221 "movq %%mm0, %%mm1 \n\t"
2222 "movq %%mm2, %%mm3 \n\t"
2223 "psrlq $24, %%mm0 \n\t"
2224 "psrlq $24, %%mm2 \n\t"
2225 PAVGB" %%mm1, %%mm0 \n\t"
2226 PAVGB" %%mm3, %%mm2 \n\t"
2227 "punpcklbw %%mm7, %%mm0 \n\t"
2228 "punpcklbw %%mm7, %%mm2 \n\t"
2230 "movd (%0, %%"REG_d"), %%mm0 \n\t"
2231 "movd (%1, %%"REG_d"), %%mm1 \n\t"
2232 "movd 3(%0, %%"REG_d"), %%mm2 \n\t"
2233 "movd 3(%1, %%"REG_d"), %%mm3 \n\t"
2234 "punpcklbw %%mm7, %%mm0 \n\t"
2235 "punpcklbw %%mm7, %%mm1 \n\t"
2236 "punpcklbw %%mm7, %%mm2 \n\t"
2237 "punpcklbw %%mm7, %%mm3 \n\t"
2238 "paddw %%mm1, %%mm0 \n\t"
2239 "paddw %%mm3, %%mm2 \n\t"
2240 "paddw %%mm2, %%mm0 \n\t"
2241 "movd 6(%0, %%"REG_d"), %%mm4 \n\t"
2242 "movd 6(%1, %%"REG_d"), %%mm1 \n\t"
2243 "movd 9(%0, %%"REG_d"), %%mm2 \n\t"
2244 "movd 9(%1, %%"REG_d"), %%mm3 \n\t"
2245 "punpcklbw %%mm7, %%mm4 \n\t"
2246 "punpcklbw %%mm7, %%mm1 \n\t"
2247 "punpcklbw %%mm7, %%mm2 \n\t"
2248 "punpcklbw %%mm7, %%mm3 \n\t"
2249 "paddw %%mm1, %%mm4 \n\t"
2250 "paddw %%mm3, %%mm2 \n\t"
2251 "paddw %%mm4, %%mm2 \n\t"
2252 "psrlw $2, %%mm0 \n\t"
2253 "psrlw $2, %%mm2 \n\t"
2255 "movq "MANGLE(bgr2VCoeff)", %%mm1 \n\t"
2256 "movq "MANGLE(bgr2VCoeff)", %%mm3 \n\t"
2258 "pmaddwd %%mm0, %%mm1 \n\t"
2259 "pmaddwd %%mm2, %%mm3 \n\t"
2260 "pmaddwd %%mm6, %%mm0 \n\t"
2261 "pmaddwd %%mm6, %%mm2 \n\t"
2262 #ifndef FAST_BGR2YV12
2263 "psrad $8, %%mm0 \n\t"
2264 "psrad $8, %%mm1 \n\t"
2265 "psrad $8, %%mm2 \n\t"
2266 "psrad $8, %%mm3 \n\t"
2268 "packssdw %%mm2, %%mm0 \n\t"
2269 "packssdw %%mm3, %%mm1 \n\t"
2270 "pmaddwd %%mm5, %%mm0 \n\t"
2271 "pmaddwd %%mm5, %%mm1 \n\t"
2272 "packssdw %%mm1, %%mm0 \n\t" // V1 V0 U1 U0
2273 "psraw $7, %%mm0 \n\t"
2275 #if defined (HAVE_MMX2) || defined (HAVE_3DNOW)
2276 "movq 12(%0, %%"REG_d"), %%mm4 \n\t"
2277 "movq 12(%1, %%"REG_d"), %%mm1 \n\t"
2278 "movq 18(%0, %%"REG_d"), %%mm2 \n\t"
2279 "movq 18(%1, %%"REG_d"), %%mm3 \n\t"
2280 PAVGB" %%mm1, %%mm4 \n\t"
2281 PAVGB" %%mm3, %%mm2 \n\t"
2282 "movq %%mm4, %%mm1 \n\t"
2283 "movq %%mm2, %%mm3 \n\t"
2284 "psrlq $24, %%mm4 \n\t"
2285 "psrlq $24, %%mm2 \n\t"
2286 PAVGB" %%mm1, %%mm4 \n\t"
2287 PAVGB" %%mm3, %%mm2 \n\t"
2288 "punpcklbw %%mm7, %%mm4 \n\t"
2289 "punpcklbw %%mm7, %%mm2 \n\t"
2291 "movd 12(%0, %%"REG_d"), %%mm4 \n\t"
2292 "movd 12(%1, %%"REG_d"), %%mm1 \n\t"
2293 "movd 15(%0, %%"REG_d"), %%mm2 \n\t"
2294 "movd 15(%1, %%"REG_d"), %%mm3 \n\t"
2295 "punpcklbw %%mm7, %%mm4 \n\t"
2296 "punpcklbw %%mm7, %%mm1 \n\t"
2297 "punpcklbw %%mm7, %%mm2 \n\t"
2298 "punpcklbw %%mm7, %%mm3 \n\t"
2299 "paddw %%mm1, %%mm4 \n\t"
2300 "paddw %%mm3, %%mm2 \n\t"
2301 "paddw %%mm2, %%mm4 \n\t"
2302 "movd 18(%0, %%"REG_d"), %%mm5 \n\t"
2303 "movd 18(%1, %%"REG_d"), %%mm1 \n\t"
2304 "movd 21(%0, %%"REG_d"), %%mm2 \n\t"
2305 "movd 21(%1, %%"REG_d"), %%mm3 \n\t"
2306 "punpcklbw %%mm7, %%mm5 \n\t"
2307 "punpcklbw %%mm7, %%mm1 \n\t"
2308 "punpcklbw %%mm7, %%mm2 \n\t"
2309 "punpcklbw %%mm7, %%mm3 \n\t"
2310 "paddw %%mm1, %%mm5 \n\t"
2311 "paddw %%mm3, %%mm2 \n\t"
2312 "paddw %%mm5, %%mm2 \n\t"
2313 "movq "MANGLE(w1111)", %%mm5 \n\t"
2314 "psrlw $2, %%mm4 \n\t"
2315 "psrlw $2, %%mm2 \n\t"
2317 "movq "MANGLE(bgr2VCoeff)", %%mm1 \n\t"
2318 "movq "MANGLE(bgr2VCoeff)", %%mm3 \n\t"
2320 "pmaddwd %%mm4, %%mm1 \n\t"
2321 "pmaddwd %%mm2, %%mm3 \n\t"
2322 "pmaddwd %%mm6, %%mm4 \n\t"
2323 "pmaddwd %%mm6, %%mm2 \n\t"
2324 #ifndef FAST_BGR2YV12
2325 "psrad $8, %%mm4 \n\t"
2326 "psrad $8, %%mm1 \n\t"
2327 "psrad $8, %%mm2 \n\t"
2328 "psrad $8, %%mm3 \n\t"
2330 "packssdw %%mm2, %%mm4 \n\t"
2331 "packssdw %%mm3, %%mm1 \n\t"
2332 "pmaddwd %%mm5, %%mm4 \n\t"
2333 "pmaddwd %%mm5, %%mm1 \n\t"
2334 "add $24, %%"REG_d" \n\t"
2335 "packssdw %%mm1, %%mm4 \n\t" // V3 V2 U3 U2
2336 "psraw $7, %%mm4 \n\t"
2338 "movq %%mm0, %%mm1 \n\t"
2339 "punpckldq %%mm4, %%mm0 \n\t"
2340 "punpckhdq %%mm4, %%mm1 \n\t"
2341 "packsswb %%mm1, %%mm0 \n\t"
2342 "paddb "MANGLE(bgr2UVOffset)", %%mm0 \n\t"
2343 "movd %%mm0, (%2, %%"REG_a") \n\t"
2344 "punpckhdq %%mm0, %%mm0 \n\t"
2345 "movd %%mm0, (%3, %%"REG_a") \n\t"
2346 "add $4, %%"REG_a" \n\t"
2348 : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth)
2349 : "%"REG_a, "%"REG_d
2352 udst += chromStride;
2353 vdst += chromStride;
2357 asm volatile( EMMS" \n\t"
2363 for(; y<height; y+=2)
2366 for(i=0; i<chromWidth; i++)
2368 unsigned int b= src[6*i+0];
2369 unsigned int g= src[6*i+1];
2370 unsigned int r= src[6*i+2];
2372 unsigned int Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2373 unsigned int V = ((RV*r + GV*g + BV*b)>>RGB2YUV_SHIFT) + 128;
2374 unsigned int U = ((RU*r + GU*g + BU*b)>>RGB2YUV_SHIFT) + 128;
2384 Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2390 for(i=0; i<chromWidth; i++)
2392 unsigned int b= src[6*i+0];
2393 unsigned int g= src[6*i+1];
2394 unsigned int r= src[6*i+2];
2396 unsigned int Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2404 Y = ((RY*r + GY*g + BY*b)>>RGB2YUV_SHIFT) + 16;
2407 udst += chromStride;
2408 vdst += chromStride;
2414 void RENAME(interleaveBytes)(uint8_t *src1, uint8_t *src2, uint8_t *dest,
2415 long width, long height, long src1Stride,
2416 long src2Stride, long dstStride){
2419 for(h=0; h < height; h++)
2426 "xor %%"REG_a", %%"REG_a" \n\t"
2428 PREFETCH" 64(%1, %%"REG_a") \n\t"
2429 PREFETCH" 64(%2, %%"REG_a") \n\t"
2430 "movdqa (%1, %%"REG_a"), %%xmm0 \n\t"
2431 "movdqa (%1, %%"REG_a"), %%xmm1 \n\t"
2432 "movdqa (%2, %%"REG_a"), %%xmm2 \n\t"
2433 "punpcklbw %%xmm2, %%xmm0 \n\t"
2434 "punpckhbw %%xmm2, %%xmm1 \n\t"
2435 "movntdq %%xmm0, (%0, %%"REG_a", 2)\n\t"
2436 "movntdq %%xmm1, 16(%0, %%"REG_a", 2)\n\t"
2437 "add $16, %%"REG_a" \n\t"
2438 "cmp %3, %%"REG_a" \n\t"
2440 ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2441 : "memory", "%"REG_a""
2445 "xor %%"REG_a", %%"REG_a" \n\t"
2447 PREFETCH" 64(%1, %%"REG_a") \n\t"
2448 PREFETCH" 64(%2, %%"REG_a") \n\t"
2449 "movq (%1, %%"REG_a"), %%mm0 \n\t"
2450 "movq 8(%1, %%"REG_a"), %%mm2 \n\t"
2451 "movq %%mm0, %%mm1 \n\t"
2452 "movq %%mm2, %%mm3 \n\t"
2453 "movq (%2, %%"REG_a"), %%mm4 \n\t"
2454 "movq 8(%2, %%"REG_a"), %%mm5 \n\t"
2455 "punpcklbw %%mm4, %%mm0 \n\t"
2456 "punpckhbw %%mm4, %%mm1 \n\t"
2457 "punpcklbw %%mm5, %%mm2 \n\t"
2458 "punpckhbw %%mm5, %%mm3 \n\t"
2459 MOVNTQ" %%mm0, (%0, %%"REG_a", 2)\n\t"
2460 MOVNTQ" %%mm1, 8(%0, %%"REG_a", 2)\n\t"
2461 MOVNTQ" %%mm2, 16(%0, %%"REG_a", 2)\n\t"
2462 MOVNTQ" %%mm3, 24(%0, %%"REG_a", 2)\n\t"
2463 "add $16, %%"REG_a" \n\t"
2464 "cmp %3, %%"REG_a" \n\t"
2466 ::"r"(dest), "r"(src1), "r"(src2), "r" (width-15)
2467 : "memory", "%"REG_a
2470 for(w= (width&(~15)); w < width; w++)
2472 dest[2*w+0] = src1[w];
2473 dest[2*w+1] = src2[w];
2476 for(w=0; w < width; w++)
2478 dest[2*w+0] = src1[w];
2479 dest[2*w+1] = src2[w];
2495 static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
2496 uint8_t *dst1, uint8_t *dst2,
2497 long width, long height,
2498 long srcStride1, long srcStride2,
2499 long dstStride1, long dstStride2)
2502 w=width/2; h=height/2;
2507 ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
2510 const uint8_t* s1=src1+srcStride1*(y>>1);
2511 uint8_t* d=dst1+dstStride1*y;
2518 "movq %1, %%mm0\n\t"
2519 "movq 8%1, %%mm2\n\t"
2520 "movq 16%1, %%mm4\n\t"
2521 "movq 24%1, %%mm6\n\t"
2522 "movq %%mm0, %%mm1\n\t"
2523 "movq %%mm2, %%mm3\n\t"
2524 "movq %%mm4, %%mm5\n\t"
2525 "movq %%mm6, %%mm7\n\t"
2526 "punpcklbw %%mm0, %%mm0\n\t"
2527 "punpckhbw %%mm1, %%mm1\n\t"
2528 "punpcklbw %%mm2, %%mm2\n\t"
2529 "punpckhbw %%mm3, %%mm3\n\t"
2530 "punpcklbw %%mm4, %%mm4\n\t"
2531 "punpckhbw %%mm5, %%mm5\n\t"
2532 "punpcklbw %%mm6, %%mm6\n\t"
2533 "punpckhbw %%mm7, %%mm7\n\t"
2534 MOVNTQ" %%mm0, %0\n\t"
2535 MOVNTQ" %%mm1, 8%0\n\t"
2536 MOVNTQ" %%mm2, 16%0\n\t"
2537 MOVNTQ" %%mm3, 24%0\n\t"
2538 MOVNTQ" %%mm4, 32%0\n\t"
2539 MOVNTQ" %%mm5, 40%0\n\t"
2540 MOVNTQ" %%mm6, 48%0\n\t"
2541 MOVNTQ" %%mm7, 56%0"
2547 for(;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
2550 const uint8_t* s2=src2+srcStride2*(y>>1);
2551 uint8_t* d=dst2+dstStride2*y;
2558 "movq %1, %%mm0\n\t"
2559 "movq 8%1, %%mm2\n\t"
2560 "movq 16%1, %%mm4\n\t"
2561 "movq 24%1, %%mm6\n\t"
2562 "movq %%mm0, %%mm1\n\t"
2563 "movq %%mm2, %%mm3\n\t"
2564 "movq %%mm4, %%mm5\n\t"
2565 "movq %%mm6, %%mm7\n\t"
2566 "punpcklbw %%mm0, %%mm0\n\t"
2567 "punpckhbw %%mm1, %%mm1\n\t"
2568 "punpcklbw %%mm2, %%mm2\n\t"
2569 "punpckhbw %%mm3, %%mm3\n\t"
2570 "punpcklbw %%mm4, %%mm4\n\t"
2571 "punpckhbw %%mm5, %%mm5\n\t"
2572 "punpcklbw %%mm6, %%mm6\n\t"
2573 "punpckhbw %%mm7, %%mm7\n\t"
2574 MOVNTQ" %%mm0, %0\n\t"
2575 MOVNTQ" %%mm1, 8%0\n\t"
2576 MOVNTQ" %%mm2, 16%0\n\t"
2577 MOVNTQ" %%mm3, 24%0\n\t"
2578 MOVNTQ" %%mm4, 32%0\n\t"
2579 MOVNTQ" %%mm5, 40%0\n\t"
2580 MOVNTQ" %%mm6, 48%0\n\t"
2581 MOVNTQ" %%mm7, 56%0"
2587 for(;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
2598 static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
2600 long width, long height,
2601 long srcStride1, long srcStride2,
2602 long srcStride3, long dstStride)
2605 w=width/2; h=height;
2607 const uint8_t* yp=src1+srcStride1*y;
2608 const uint8_t* up=src2+srcStride2*(y>>2);
2609 const uint8_t* vp=src3+srcStride3*(y>>2);
2610 uint8_t* d=dst+dstStride*y;
2616 PREFETCH" 32(%1, %0)\n\t"
2617 PREFETCH" 32(%2, %0)\n\t"
2618 PREFETCH" 32(%3, %0)\n\t"
2619 "movq (%1, %0, 4), %%mm0\n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2620 "movq (%2, %0), %%mm1\n\t" /* U0U1U2U3U4U5U6U7 */
2621 "movq (%3, %0), %%mm2\n\t" /* V0V1V2V3V4V5V6V7 */
2622 "movq %%mm0, %%mm3\n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
2623 "movq %%mm1, %%mm4\n\t" /* U0U1U2U3U4U5U6U7 */
2624 "movq %%mm2, %%mm5\n\t" /* V0V1V2V3V4V5V6V7 */
2625 "punpcklbw %%mm1, %%mm1\n\t" /* U0U0 U1U1 U2U2 U3U3 */
2626 "punpcklbw %%mm2, %%mm2\n\t" /* V0V0 V1V1 V2V2 V3V3 */
2627 "punpckhbw %%mm4, %%mm4\n\t" /* U4U4 U5U5 U6U6 U7U7 */
2628 "punpckhbw %%mm5, %%mm5\n\t" /* V4V4 V5V5 V6V6 V7V7 */
2630 "movq %%mm1, %%mm6\n\t"
2631 "punpcklbw %%mm2, %%mm1\n\t" /* U0V0 U0V0 U1V1 U1V1*/
2632 "punpcklbw %%mm1, %%mm0\n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
2633 "punpckhbw %%mm1, %%mm3\n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
2634 MOVNTQ" %%mm0, (%4, %0, 8)\n\t"
2635 MOVNTQ" %%mm3, 8(%4, %0, 8)\n\t"
2637 "punpckhbw %%mm2, %%mm6\n\t" /* U2V2 U2V2 U3V3 U3V3*/
2638 "movq 8(%1, %0, 4), %%mm0\n\t"
2639 "movq %%mm0, %%mm3\n\t"
2640 "punpcklbw %%mm6, %%mm0\n\t" /* Y U2 Y V2 Y U2 Y V2*/
2641 "punpckhbw %%mm6, %%mm3\n\t" /* Y U3 Y V3 Y U3 Y V3*/
2642 MOVNTQ" %%mm0, 16(%4, %0, 8)\n\t"
2643 MOVNTQ" %%mm3, 24(%4, %0, 8)\n\t"
2645 "movq %%mm4, %%mm6\n\t"
2646 "movq 16(%1, %0, 4), %%mm0\n\t"
2647 "movq %%mm0, %%mm3\n\t"
2648 "punpcklbw %%mm5, %%mm4\n\t"
2649 "punpcklbw %%mm4, %%mm0\n\t" /* Y U4 Y V4 Y U4 Y V4*/
2650 "punpckhbw %%mm4, %%mm3\n\t" /* Y U5 Y V5 Y U5 Y V5*/
2651 MOVNTQ" %%mm0, 32(%4, %0, 8)\n\t"
2652 MOVNTQ" %%mm3, 40(%4, %0, 8)\n\t"
2654 "punpckhbw %%mm5, %%mm6\n\t"
2655 "movq 24(%1, %0, 4), %%mm0\n\t"
2656 "movq %%mm0, %%mm3\n\t"
2657 "punpcklbw %%mm6, %%mm0\n\t" /* Y U6 Y V6 Y U6 Y V6*/
2658 "punpckhbw %%mm6, %%mm3\n\t" /* Y U7 Y V7 Y U7 Y V7*/
2659 MOVNTQ" %%mm0, 48(%4, %0, 8)\n\t"
2660 MOVNTQ" %%mm3, 56(%4, %0, 8)\n\t"
2663 : "r"(yp), "r" (up), "r"(vp), "r"(d)
2669 const long x2= x<<2;