8df289457cab6579039b65a8fe5666414c6c3c5e
[x264/x264-sandbox.git] / common / ppc / mc.c
1 /*****************************************************************************
2  * mc.c: ppc motion compensation
3  *****************************************************************************
4  * Copyright (C) 2003-2016 x264 project
5  *
6  * Authors: Eric Petit <eric.petit@lapsus.org>
7  *          Guillaume Poirier <gpoirier@mplayerhq.hu>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this program; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
22  *
23  * This program is also available under a commercial proprietary license.
24  * For more information, contact us at licensing@x264.com.
25  *****************************************************************************/
26
27 #include <stdlib.h>
28 #include <stdio.h>
29 #include <string.h>
30 #include <stdint.h>
31 #include <stdarg.h>
32
33 #include "x264.h"
34 #include "common/common.h"
35 #include "common/mc.h"
36 #include "mc.h"
37 #include "ppccommon.h"
38
39 #if !HIGH_BIT_DEPTH
40 typedef void (*pf_mc_t)( uint8_t *src, intptr_t i_src,
41                          uint8_t *dst, intptr_t i_dst, int i_height );
42
43 static inline int x264_tapfilter( uint8_t *pix, int i_pix_next )
44 {
45     return pix[-2*i_pix_next] - 5*pix[-1*i_pix_next] + 20*(pix[0] +
46            pix[1*i_pix_next]) - 5*pix[ 2*i_pix_next] +
47            pix[ 3*i_pix_next];
48 }
49
50 static inline int x264_tapfilter1( uint8_t *pix )
51 {
52     return pix[-2] - 5*pix[-1] + 20*(pix[0] + pix[1]) - 5*pix[ 2] +
53            pix[ 3];
54 }
55
56 static inline void x264_pixel_avg2_w4_altivec( uint8_t *dst,  intptr_t i_dst,
57                                                uint8_t *src1, intptr_t i_src1,
58                                                uint8_t *src2, int i_height )
59 {
60     for( int y = 0; y < i_height; y++ )
61     {
62         for( int x = 0; x < 4; x++ )
63             dst[x] = ( src1[x] + src2[x] + 1 ) >> 1;
64         dst  += i_dst;
65         src1 += i_src1;
66         src2 += i_src1;
67     }
68 }
69
70 static inline void x264_pixel_avg2_w8_altivec( uint8_t *dst,  intptr_t i_dst,
71                                                uint8_t *src1, intptr_t i_src1,
72                                                uint8_t *src2, int i_height )
73 {
74     vec_u8_t src1v, src2v;
75     PREP_LOAD;
76     PREP_STORE8;
77     PREP_LOAD_SRC( src1 );
78     PREP_LOAD_SRC( src2 );
79
80     for( int y = 0; y < i_height; y++ )
81     {
82         VEC_LOAD( src1, src1v, 8, vec_u8_t, src1 );
83         VEC_LOAD( src2, src2v, 8, vec_u8_t, src2 );
84         src1v = vec_avg( src1v, src2v );
85         VEC_STORE8( src1v, dst );
86
87         dst  += i_dst;
88         src1 += i_src1;
89         src2 += i_src1;
90     }
91 }
92
93 static inline void x264_pixel_avg2_w16_altivec( uint8_t *dst,  intptr_t i_dst,
94                                                 uint8_t *src1, intptr_t i_src1,
95                                                 uint8_t *src2, int i_height )
96 {
97     vec_u8_t src1v, src2v;
98     PREP_LOAD;
99     PREP_LOAD_SRC( src1 );
100     PREP_LOAD_SRC( src2 );
101
102     for( int y = 0; y < i_height; y++ )
103     {
104         VEC_LOAD( src1, src1v, 16, vec_u8_t, src1 );
105         VEC_LOAD( src2, src2v, 16, vec_u8_t, src2 );
106         src1v = vec_avg( src1v, src2v );
107         vec_st(src1v, 0, dst);
108
109         dst  += i_dst;
110         src1 += i_src1;
111         src2 += i_src1;
112     }
113 }
114
115 static inline void x264_pixel_avg2_w20_altivec( uint8_t *dst,  intptr_t i_dst,
116                                                 uint8_t *src1, intptr_t i_src1,
117                                                 uint8_t *src2, int i_height )
118 {
119     x264_pixel_avg2_w16_altivec(dst, i_dst, src1, i_src1, src2, i_height);
120     x264_pixel_avg2_w4_altivec(dst+16, i_dst, src1+16, i_src1, src2+16, i_height);
121 }
122
123 /* mc_copy: plain c */
124
125 #define MC_COPY( name, a )                                \
126 static void name( uint8_t *dst, intptr_t i_dst,           \
127                   uint8_t *src, intptr_t i_src, int i_height ) \
128 {                                                         \
129     int y;                                                \
130     for( y = 0; y < i_height; y++ )                       \
131     {                                                     \
132         memcpy( dst, src, a );                            \
133         src += i_src;                                     \
134         dst += i_dst;                                     \
135     }                                                     \
136 }
137 MC_COPY( x264_mc_copy_w4_altivec,  4  )
138 MC_COPY( x264_mc_copy_w8_altivec,  8  )
139
140 static void x264_mc_copy_w16_altivec( uint8_t *dst, intptr_t i_dst,
141                                       uint8_t *src, intptr_t i_src, int i_height )
142 {
143     vec_u8_t cpyV;
144     PREP_LOAD;
145     PREP_LOAD_SRC( src );
146
147     for( int y = 0; y < i_height; y++ )
148     {
149         VEC_LOAD( src, cpyV, 16, vec_u8_t, src );
150         vec_st(cpyV, 0, dst);
151
152         src += i_src;
153         dst += i_dst;
154     }
155 }
156
157
158 static void x264_mc_copy_w16_aligned_altivec( uint8_t *dst, intptr_t i_dst,
159                                               uint8_t *src, intptr_t i_src, int i_height )
160 {
161     for( int y = 0; y < i_height; ++y )
162     {
163         vec_u8_t cpyV = vec_ld( 0, src );
164         vec_st(cpyV, 0, dst);
165
166         src += i_src;
167         dst += i_dst;
168     }
169 }
170
171
172 static void mc_luma_altivec( uint8_t *dst,    intptr_t i_dst_stride,
173                              uint8_t *src[4], intptr_t i_src_stride,
174                              int mvx, int mvy,
175                              int i_width, int i_height, const x264_weight_t *weight )
176 {
177     int qpel_idx = ((mvy&3)<<2) + (mvx&3);
178     intptr_t offset = (mvy>>2)*i_src_stride + (mvx>>2);
179     uint8_t *src1 = src[x264_hpel_ref0[qpel_idx]] + offset + ((mvy&3) == 3) * i_src_stride;
180     if( qpel_idx & 5 ) /* qpel interpolation needed */
181     {
182         uint8_t *src2 = src[x264_hpel_ref1[qpel_idx]] + offset + ((mvx&3) == 3);
183
184         switch( i_width )
185         {
186             case 4:
187                 x264_pixel_avg2_w4_altivec( dst, i_dst_stride, src1, i_src_stride, src2, i_height );
188                 break;
189             case 8:
190                 x264_pixel_avg2_w8_altivec( dst, i_dst_stride, src1, i_src_stride, src2, i_height );
191                 break;
192             case 16:
193             default:
194                 x264_pixel_avg2_w16_altivec( dst, i_dst_stride, src1, i_src_stride, src2, i_height );
195         }
196         if( weight->weightfn )
197             weight->weightfn[i_width>>2]( dst, i_dst_stride, dst, i_dst_stride, weight, i_height );
198     }
199     else if( weight->weightfn )
200         weight->weightfn[i_width>>2]( dst, i_dst_stride, src1, i_src_stride, weight, i_height );
201     else
202     {
203         switch( i_width )
204         {
205             case 4:
206                 x264_mc_copy_w4_altivec( dst, i_dst_stride, src1, i_src_stride, i_height );
207                 break;
208             case 8:
209                 x264_mc_copy_w8_altivec( dst, i_dst_stride, src1, i_src_stride, i_height );
210                 break;
211             case 16:
212                 x264_mc_copy_w16_altivec( dst, i_dst_stride, src1, i_src_stride, i_height );
213                 break;
214         }
215     }
216 }
217
218
219
220 static uint8_t *get_ref_altivec( uint8_t *dst,   intptr_t *i_dst_stride,
221                                  uint8_t *src[4], intptr_t i_src_stride,
222                                  int mvx, int mvy,
223                                  int i_width, int i_height, const x264_weight_t *weight )
224 {
225     int qpel_idx = ((mvy&3)<<2) + (mvx&3);
226     intptr_t offset = (mvy>>2)*i_src_stride + (mvx>>2);
227     uint8_t *src1 = src[x264_hpel_ref0[qpel_idx]] + offset + ((mvy&3) == 3) * i_src_stride;
228     if( qpel_idx & 5 ) /* qpel interpolation needed */
229     {
230         uint8_t *src2 = src[x264_hpel_ref1[qpel_idx]] + offset + ((mvx&3) == 3);
231         switch( i_width )
232         {
233             case 4:
234                 x264_pixel_avg2_w4_altivec( dst, *i_dst_stride, src1, i_src_stride, src2, i_height );
235                 break;
236             case 8:
237                 x264_pixel_avg2_w8_altivec( dst, *i_dst_stride, src1, i_src_stride, src2, i_height );
238                 break;
239             case 12:
240             case 16:
241             default:
242                 x264_pixel_avg2_w16_altivec( dst, *i_dst_stride, src1, i_src_stride, src2, i_height );
243                 break;
244             case 20:
245                 x264_pixel_avg2_w20_altivec( dst, *i_dst_stride, src1, i_src_stride, src2, i_height );
246                 break;
247         }
248         if( weight->weightfn )
249             weight->weightfn[i_width>>2]( dst, *i_dst_stride, dst, *i_dst_stride, weight, i_height );
250         return dst;
251     }
252     else if( weight->weightfn )
253     {
254         weight->weightfn[i_width>>2]( dst, *i_dst_stride, src1, i_src_stride, weight, i_height );
255         return dst;
256     }
257     else
258     {
259         *i_dst_stride = i_src_stride;
260         return src1;
261     }
262 }
263
264 static void mc_chroma_2xh( uint8_t *dstu, uint8_t *dstv, intptr_t i_dst_stride,
265                            uint8_t *src, intptr_t i_src_stride,
266                            int mvx, int mvy, int i_height )
267 {
268     uint8_t *srcp;
269     int d8x = mvx&0x07;
270     int d8y = mvy&0x07;
271
272     int cA = (8-d8x)*(8-d8y);
273     int cB = d8x    *(8-d8y);
274     int cC = (8-d8x)*d8y;
275     int cD = d8x    *d8y;
276
277     src += (mvy >> 3) * i_src_stride + (mvx >> 3)*2;
278     srcp = &src[i_src_stride];
279
280     for( int y = 0; y < i_height; y++ )
281     {
282         dstu[0] = ( cA*src[0] + cB*src[2] + cC*srcp[0] + cD*srcp[2] + 32 ) >> 6;
283         dstv[0] = ( cA*src[1] + cB*src[3] + cC*srcp[1] + cD*srcp[3] + 32 ) >> 6;
284         dstu[1] = ( cA*src[2] + cB*src[4] + cC*srcp[2] + cD*srcp[4] + 32 ) >> 6;
285         dstv[1] = ( cA*src[3] + cB*src[5] + cC*srcp[3] + cD*srcp[5] + 32 ) >> 6;
286
287         src  += i_src_stride;
288         srcp += i_src_stride;
289         dstu += i_dst_stride;
290         dstv += i_dst_stride;
291     }
292  }
293
294 #ifdef WORDS_BIGENDIAN
295 #define VSLD(a,b,n) vec_sld(a,b,n)
296 #else
297 #define VSLD(a,b,n) vec_sld(b,a,16-n)
298 #endif
299
300 static void mc_chroma_altivec_4xh( uint8_t *dstu, uint8_t *dstv, intptr_t i_dst_stride,
301                                    uint8_t *src, intptr_t i_src_stride,
302                                    int mvx, int mvy, int i_height )
303 {
304     uint8_t *srcp;
305     int d8x = mvx & 0x07;
306     int d8y = mvy & 0x07;
307
308     ALIGNED_16( uint16_t coeff[4] );
309     coeff[0] = (8-d8x)*(8-d8y);
310     coeff[1] = d8x    *(8-d8y);
311     coeff[2] = (8-d8x)*d8y;
312     coeff[3] = d8x    *d8y;
313
314     src += (mvy >> 3) * i_src_stride + (mvx >> 3)*2;
315     srcp = &src[i_src_stride];
316
317     LOAD_ZERO;
318     PREP_LOAD;
319     PREP_LOAD_SRC( src );
320     vec_u16_t   coeff0v, coeff1v, coeff2v, coeff3v;
321     vec_u8_t    src2v_8, dstuv, dstvv;
322     vec_u16_t   src0v_16, src1v_16, src2v_16, src3v_16, dstv16;
323     vec_u16_t   shiftv, k32v;
324
325 #ifdef WORDS_BIGENDIAN
326     static const vec_u8_t perm0v = CV(1,5,9,13,1,5,9,13,1,5,9,13,1,5,9,13);
327     static const vec_u8_t perm1v = CV(3,7,11,15,3,7,11,15,3,7,11,15,3,7,11,15);
328 #else
329     static const vec_u8_t perm0v = CV(0,4,8,12,0,4,8,12,0,4,8,12,0,4,8,12);
330     static const vec_u8_t perm1v = CV(2,6,10,14,2,6,10,14,2,6,10,14,2,6,10,14);
331 #endif
332
333     coeff0v = vec_ld( 0, coeff );
334     coeff3v = vec_splat( coeff0v, 3 );
335     coeff2v = vec_splat( coeff0v, 2 );
336     coeff1v = vec_splat( coeff0v, 1 );
337     coeff0v = vec_splat( coeff0v, 0 );
338     k32v    = vec_sl( vec_splat_u16( 1 ), vec_splat_u16( 5 ) );
339     shiftv  = vec_splat_u16( 6 );
340
341     VEC_LOAD( src, src2v_8, 9, vec_u8_t, src );
342     src2v_16 = vec_u8_to_u16( src2v_8 );
343     src3v_16 = vec_u8_to_u16( VSLD( src2v_8, src2v_8, 2 ) );
344
345     for( int y = 0; y < i_height; y += 2 )
346     {
347         src0v_16 = src2v_16;
348         src1v_16 = src3v_16;
349         VEC_LOAD( srcp, src2v_8, 9, vec_u8_t, src );
350         src2v_16 = vec_u8_to_u16( src2v_8 );
351         src3v_16 = vec_u8_to_u16( VSLD( src2v_8, src2v_8, 2 ) );
352
353         dstv16 = vec_mladd( coeff0v, src0v_16, k32v );
354         dstv16 = vec_mladd( coeff1v, src1v_16, dstv16 );
355         dstv16 = vec_mladd( coeff2v, src2v_16, dstv16 );
356         dstv16 = vec_mladd( coeff3v, src3v_16, dstv16 );
357
358         dstv16 = vec_sr( dstv16, shiftv );
359
360         dstuv = (vec_u8_t)vec_perm( dstv16, dstv16, perm0v );
361         dstvv = (vec_u8_t)vec_perm( dstv16, dstv16, perm1v );
362         vec_ste( (vec_u32_t)dstuv, 0, (uint32_t*) dstu );
363         vec_ste( (vec_u32_t)dstvv, 0, (uint32_t*) dstv );
364
365         srcp += i_src_stride;
366         dstu += i_dst_stride;
367         dstv += i_dst_stride;
368
369         src0v_16 = src2v_16;
370         src1v_16 = src3v_16;
371         VEC_LOAD( srcp, src2v_8, 9, vec_u8_t, src );
372         src2v_16 = vec_u8_to_u16( src2v_8 );
373         src3v_16 = vec_u8_to_u16( VSLD( src2v_8, src2v_8, 2 ) );
374
375         dstv16 = vec_mladd( coeff0v, src0v_16, k32v );
376         dstv16 = vec_mladd( coeff1v, src1v_16, dstv16 );
377         dstv16 = vec_mladd( coeff2v, src2v_16, dstv16 );
378         dstv16 = vec_mladd( coeff3v, src3v_16, dstv16 );
379
380         dstv16 = vec_sr( dstv16, shiftv );
381
382         dstuv = (vec_u8_t)vec_perm( dstv16, dstv16, perm0v );
383         dstvv = (vec_u8_t)vec_perm( dstv16, dstv16, perm1v );
384         vec_ste( (vec_u32_t)dstuv, 0, (uint32_t*) dstu );
385         vec_ste( (vec_u32_t)dstvv, 0, (uint32_t*) dstv );
386
387         srcp += i_src_stride;
388         dstu += i_dst_stride;
389         dstv += i_dst_stride;
390     }
391 }
392
393 static void mc_chroma_altivec_8xh( uint8_t *dstu, uint8_t *dstv, intptr_t i_dst_stride,
394                                    uint8_t *src, intptr_t i_src_stride,
395                                    int mvx, int mvy, int i_height )
396 {
397     uint8_t *srcp;
398     int d8x = mvx & 0x07;
399     int d8y = mvy & 0x07;
400
401     ALIGNED_16( uint16_t coeff[4] );
402     coeff[0] = (8-d8x)*(8-d8y);
403     coeff[1] = d8x    *(8-d8y);
404     coeff[2] = (8-d8x)*d8y;
405     coeff[3] = d8x    *d8y;
406
407     src += (mvy >> 3) * i_src_stride + (mvx >> 3)*2;
408     srcp = &src[i_src_stride];
409
410     LOAD_ZERO;
411     PREP_LOAD;
412     PREP_LOAD_SRC( src );
413     PREP_STORE8;
414     vec_u16_t   coeff0v, coeff1v, coeff2v, coeff3v;
415     vec_u8_t    src0v_8, src1v_8, src2v_8, src3v_8;
416     vec_u8_t    dstuv, dstvv;
417     vec_u16_t   src0v_16h, src1v_16h, src2v_16h, src3v_16h, dstv_16h;
418     vec_u16_t   src0v_16l, src1v_16l, src2v_16l, src3v_16l, dstv_16l;
419     vec_u16_t   shiftv, k32v;
420
421     coeff0v = vec_ld( 0, coeff );
422     coeff3v = vec_splat( coeff0v, 3 );
423     coeff2v = vec_splat( coeff0v, 2 );
424     coeff1v = vec_splat( coeff0v, 1 );
425     coeff0v = vec_splat( coeff0v, 0 );
426     k32v    = vec_sl( vec_splat_u16( 1 ), vec_splat_u16( 5 ) );
427     shiftv  = vec_splat_u16( 6 );
428
429 #ifdef WORDS_BIGENDIAN
430     static const vec_u8_t perm0v = CV(1,5,9,13,17,21,25,29,0,0,0,0,0,0,0,0);
431     static const vec_u8_t perm1v = CV(3,7,11,15,19,23,27,31,0,0,0,0,0,0,0,0);
432 #else
433     static const vec_u8_t perm0v = CV(0,4,8,12,16,20,24,28,1,1,1,1,1,1,1,1);
434     static const vec_u8_t perm1v = CV(2,6,10,14,18,22,26,30,1,1,1,1,1,1,1,1);
435 #endif
436
437     VEC_LOAD( src, src2v_8, 16, vec_u8_t, src );
438     VEC_LOAD( src+16, src3v_8, 2, vec_u8_t, src );
439     src3v_8 = VSLD( src2v_8, src3v_8, 2 );
440
441     for( int y = 0; y < i_height; y += 2 )
442     {
443         src0v_8 = src2v_8;
444         src1v_8 = src3v_8;
445         VEC_LOAD( srcp, src2v_8, 16, vec_u8_t, src );
446         VEC_LOAD( srcp+16, src3v_8, 2, vec_u8_t, src );
447
448         src3v_8 = VSLD( src2v_8, src3v_8, 2 );
449
450         src0v_16h = vec_u8_to_u16_h( src0v_8 );
451         src0v_16l = vec_u8_to_u16_l( src0v_8 );
452         src1v_16h = vec_u8_to_u16_h( src1v_8 );
453         src1v_16l = vec_u8_to_u16_l( src1v_8 );
454         src2v_16h = vec_u8_to_u16_h( src2v_8 );
455         src2v_16l = vec_u8_to_u16_l( src2v_8 );
456         src3v_16h = vec_u8_to_u16_h( src3v_8 );
457         src3v_16l = vec_u8_to_u16_l( src3v_8 );
458
459         dstv_16h = vec_mladd( coeff0v, src0v_16h, k32v );
460         dstv_16l = vec_mladd( coeff0v, src0v_16l, k32v );
461         dstv_16h = vec_mladd( coeff1v, src1v_16h, dstv_16h );
462         dstv_16l = vec_mladd( coeff1v, src1v_16l, dstv_16l );
463         dstv_16h = vec_mladd( coeff2v, src2v_16h, dstv_16h );
464         dstv_16l = vec_mladd( coeff2v, src2v_16l, dstv_16l );
465         dstv_16h = vec_mladd( coeff3v, src3v_16h, dstv_16h );
466         dstv_16l = vec_mladd( coeff3v, src3v_16l, dstv_16l );
467
468         dstv_16h = vec_sr( dstv_16h, shiftv );
469         dstv_16l = vec_sr( dstv_16l, shiftv );
470
471         dstuv = (vec_u8_t)vec_perm( dstv_16h, dstv_16l, perm0v );
472         dstvv = (vec_u8_t)vec_perm( dstv_16h, dstv_16l, perm1v );
473
474         VEC_STORE8( dstuv, dstu );
475         VEC_STORE8( dstvv, dstv );
476
477         srcp += i_src_stride;
478         dstu += i_dst_stride;
479         dstv += i_dst_stride;
480
481         src0v_8 = src2v_8;
482         src1v_8 = src3v_8;
483         VEC_LOAD( srcp, src2v_8, 16, vec_u8_t, src );
484         VEC_LOAD( srcp+16, src3v_8, 2, vec_u8_t, src );
485
486         src3v_8 = VSLD( src2v_8, src3v_8, 2 );
487
488         src0v_16h = vec_u8_to_u16_h( src0v_8 );
489         src0v_16l = vec_u8_to_u16_l( src0v_8 );
490         src1v_16h = vec_u8_to_u16_h( src1v_8 );
491         src1v_16l = vec_u8_to_u16_l( src1v_8 );
492         src2v_16h = vec_u8_to_u16_h( src2v_8 );
493         src2v_16l = vec_u8_to_u16_l( src2v_8 );
494         src3v_16h = vec_u8_to_u16_h( src3v_8 );
495         src3v_16l = vec_u8_to_u16_l( src3v_8 );
496
497         dstv_16h = vec_mladd( coeff0v, src0v_16h, k32v );
498         dstv_16l = vec_mladd( coeff0v, src0v_16l, k32v );
499         dstv_16h = vec_mladd( coeff1v, src1v_16h, dstv_16h );
500         dstv_16l = vec_mladd( coeff1v, src1v_16l, dstv_16l );
501         dstv_16h = vec_mladd( coeff2v, src2v_16h, dstv_16h );
502         dstv_16l = vec_mladd( coeff2v, src2v_16l, dstv_16l );
503         dstv_16h = vec_mladd( coeff3v, src3v_16h, dstv_16h );
504         dstv_16l = vec_mladd( coeff3v, src3v_16l, dstv_16l );
505
506         dstv_16h = vec_sr( dstv_16h, shiftv );
507         dstv_16l = vec_sr( dstv_16l, shiftv );
508
509         dstuv = (vec_u8_t)vec_perm( dstv_16h, dstv_16l, perm0v );
510         dstvv = (vec_u8_t)vec_perm( dstv_16h, dstv_16l, perm1v );
511
512         VEC_STORE8( dstuv, dstu );
513         VEC_STORE8( dstvv, dstv );
514
515         srcp += i_src_stride;
516         dstu += i_dst_stride;
517         dstv += i_dst_stride;
518     }
519 }
520
521 static void mc_chroma_altivec( uint8_t *dstu, uint8_t *dstv, intptr_t i_dst_stride,
522                                uint8_t *src, intptr_t i_src_stride,
523                                int mvx, int mvy, int i_width, int i_height )
524 {
525     if( i_width == 8 )
526         mc_chroma_altivec_8xh( dstu, dstv, i_dst_stride, src, i_src_stride,
527                                mvx, mvy, i_height );
528     else if( i_width == 4 )
529         mc_chroma_altivec_4xh( dstu, dstv, i_dst_stride, src, i_src_stride,
530                                mvx, mvy, i_height );
531     else
532         mc_chroma_2xh( dstu, dstv, i_dst_stride, src, i_src_stride,
533                        mvx, mvy, i_height );
534 }
535
536 #define HPEL_FILTER_1( t1v, t2v, t3v, t4v, t5v, t6v ) \
537 {                                                     \
538     t1v = vec_add( t1v, t6v );                        \
539     t2v = vec_add( t2v, t5v );                        \
540     t3v = vec_add( t3v, t4v );                        \
541                                                       \
542     t1v = vec_sub( t1v, t2v );   /* (a-b) */          \
543     t2v = vec_sub( t2v, t3v );   /* (b-c) */          \
544     t2v = vec_sl(  t2v, twov );  /* (b-c)*4 */        \
545     t1v = vec_sub( t1v, t2v );   /* a-5*b+4*c */      \
546     t3v = vec_sl(  t3v, fourv ); /* 16*c */           \
547     t1v = vec_add( t1v, t3v );   /* a-5*b+20*c */     \
548 }
549
550 #define HPEL_FILTER_2( t1v, t2v, t3v, t4v, t5v, t6v ) \
551 {                                                     \
552     t1v = vec_add( t1v, t6v );                        \
553     t2v = vec_add( t2v, t5v );                        \
554     t3v = vec_add( t3v, t4v );                        \
555                                                       \
556     t1v = vec_sub( t1v, t2v );  /* (a-b) */           \
557     t1v = vec_sra( t1v, twov ); /* (a-b)/4 */         \
558     t1v = vec_sub( t1v, t2v );  /* (a-b)/4-b */       \
559     t1v = vec_add( t1v, t3v );  /* (a-b)/4-b+c */     \
560     t1v = vec_sra( t1v, twov ); /* ((a-b)/4-b+c)/4 */ \
561     t1v = vec_add( t1v, t3v );  /* ((a-b)/4-b+c)/4+c = (a-5*b+20*c)/16 */ \
562 }
563
564 #define HPEL_FILTER_HORIZONTAL()                             \
565 {                                                            \
566     VEC_LOAD_G( &src[x- 2+i_stride*y], src1v, 16, vec_u8_t); \
567     VEC_LOAD_G( &src[x+14+i_stride*y], src6v, 16, vec_u8_t); \
568                                                              \
569     src2v = VSLD( src1v, src6v,  1 );                        \
570     src3v = VSLD( src1v, src6v,  2 );                        \
571     src4v = VSLD( src1v, src6v,  3 );                        \
572     src5v = VSLD( src1v, src6v,  4 );                        \
573     src6v = VSLD( src1v, src6v,  5 );                        \
574                                                              \
575     temp1v = vec_u8_to_s16_h( src1v );                       \
576     temp2v = vec_u8_to_s16_h( src2v );                       \
577     temp3v = vec_u8_to_s16_h( src3v );                       \
578     temp4v = vec_u8_to_s16_h( src4v );                       \
579     temp5v = vec_u8_to_s16_h( src5v );                       \
580     temp6v = vec_u8_to_s16_h( src6v );                       \
581                                                              \
582     HPEL_FILTER_1( temp1v, temp2v, temp3v,                   \
583                    temp4v, temp5v, temp6v );                 \
584                                                              \
585     dest1v = vec_add( temp1v, sixteenv );                    \
586     dest1v = vec_sra( dest1v, fivev );                       \
587                                                              \
588     temp1v = vec_u8_to_s16_l( src1v );                       \
589     temp2v = vec_u8_to_s16_l( src2v );                       \
590     temp3v = vec_u8_to_s16_l( src3v );                       \
591     temp4v = vec_u8_to_s16_l( src4v );                       \
592     temp5v = vec_u8_to_s16_l( src5v );                       \
593     temp6v = vec_u8_to_s16_l( src6v );                       \
594                                                              \
595     HPEL_FILTER_1( temp1v, temp2v, temp3v,                   \
596                    temp4v, temp5v, temp6v );                 \
597                                                              \
598     dest2v = vec_add( temp1v, sixteenv );                    \
599     dest2v = vec_sra( dest2v, fivev );                       \
600                                                              \
601     destv = vec_packsu( dest1v, dest2v );                    \
602                                                              \
603     VEC_STORE16( destv, &dsth[x+i_stride*y], dsth );         \
604 }
605
606 #define HPEL_FILTER_VERTICAL()                                    \
607 {                                                                 \
608     VEC_LOAD( &src[x+i_stride*(y-2)], src1v, 16, vec_u8_t, src ); \
609     VEC_LOAD( &src[x+i_stride*(y-1)], src2v, 16, vec_u8_t, src ); \
610     VEC_LOAD( &src[x+i_stride*(y-0)], src3v, 16, vec_u8_t, src ); \
611     VEC_LOAD( &src[x+i_stride*(y+1)], src4v, 16, vec_u8_t, src ); \
612     VEC_LOAD( &src[x+i_stride*(y+2)], src5v, 16, vec_u8_t, src ); \
613     VEC_LOAD( &src[x+i_stride*(y+3)], src6v, 16, vec_u8_t, src ); \
614                                                                   \
615     temp1v = vec_u8_to_s16_h( src1v );                            \
616     temp2v = vec_u8_to_s16_h( src2v );                            \
617     temp3v = vec_u8_to_s16_h( src3v );                            \
618     temp4v = vec_u8_to_s16_h( src4v );                            \
619     temp5v = vec_u8_to_s16_h( src5v );                            \
620     temp6v = vec_u8_to_s16_h( src6v );                            \
621                                                                   \
622     HPEL_FILTER_1( temp1v, temp2v, temp3v,                        \
623                    temp4v, temp5v, temp6v );                      \
624                                                                   \
625     dest1v = vec_add( temp1v, sixteenv );                         \
626     dest1v = vec_sra( dest1v, fivev );                            \
627                                                                   \
628     temp4v = vec_u8_to_s16_l( src1v );                            \
629     temp5v = vec_u8_to_s16_l( src2v );                            \
630     temp6v = vec_u8_to_s16_l( src3v );                            \
631     temp7v = vec_u8_to_s16_l( src4v );                            \
632     temp8v = vec_u8_to_s16_l( src5v );                            \
633     temp9v = vec_u8_to_s16_l( src6v );                            \
634                                                                   \
635     HPEL_FILTER_1( temp4v, temp5v, temp6v,                        \
636                    temp7v, temp8v, temp9v );                      \
637                                                                   \
638     dest2v = vec_add( temp4v, sixteenv );                         \
639     dest2v = vec_sra( dest2v, fivev );                            \
640                                                                   \
641     destv = vec_packsu( dest1v, dest2v );                         \
642                                                                   \
643     VEC_STORE16( destv, &dstv[x+i_stride*y], dsth );              \
644 }
645
646 #define HPEL_FILTER_CENTRAL()                           \
647 {                                                       \
648     temp1v = VSLD( tempav, tempbv, 12 );                \
649     temp2v = VSLD( tempav, tempbv, 14 );                \
650     temp3v = tempbv;                                    \
651     temp4v = VSLD( tempbv, tempcv,  2 );                \
652     temp5v = VSLD( tempbv, tempcv,  4 );                \
653     temp6v = VSLD( tempbv, tempcv,  6 );                \
654                                                         \
655     HPEL_FILTER_2( temp1v, temp2v, temp3v,              \
656                    temp4v, temp5v, temp6v );            \
657                                                         \
658     dest1v = vec_add( temp1v, thirtytwov );             \
659     dest1v = vec_sra( dest1v, sixv );                   \
660                                                         \
661     temp1v = VSLD( tempbv, tempcv, 12 );                \
662     temp2v = VSLD( tempbv, tempcv, 14 );                \
663     temp3v = tempcv;                                    \
664     temp4v = VSLD( tempcv, tempdv,  2 );                \
665     temp5v = VSLD( tempcv, tempdv,  4 );                \
666     temp6v = VSLD( tempcv, tempdv,  6 );                \
667                                                         \
668     HPEL_FILTER_2( temp1v, temp2v, temp3v,              \
669                    temp4v, temp5v, temp6v );            \
670                                                         \
671     dest2v = vec_add( temp1v, thirtytwov );             \
672     dest2v = vec_sra( dest2v, sixv );                   \
673                                                         \
674     destv = vec_packsu( dest1v, dest2v );               \
675                                                         \
676     VEC_STORE16( destv, &dstc[x-16+i_stride*y], dsth ); \
677 }
678
679 void x264_hpel_filter_altivec( uint8_t *dsth, uint8_t *dstv, uint8_t *dstc, uint8_t *src,
680                                intptr_t i_stride, int i_width, int i_height, int16_t *buf )
681 {
682     vec_u8_t destv;
683     vec_u8_t src1v, src2v, src3v, src4v, src5v, src6v;
684     vec_s16_t dest1v, dest2v;
685     vec_s16_t temp1v, temp2v, temp3v, temp4v, temp5v, temp6v, temp7v, temp8v, temp9v;
686     vec_s16_t tempav, tempbv, tempcv, tempdv, tempev;
687
688     PREP_LOAD;
689     PREP_LOAD_SRC( src);
690     PREP_STORE16;
691     PREP_STORE16_DST( dsth );
692     LOAD_ZERO;
693
694     vec_u16_t twov, fourv, fivev, sixv;
695     vec_s16_t sixteenv, thirtytwov;
696     vec_u16_u temp_u;
697
698     temp_u.s[0]=2;
699     twov = vec_splat( temp_u.v, 0 );
700     temp_u.s[0]=4;
701     fourv = vec_splat( temp_u.v, 0 );
702     temp_u.s[0]=5;
703     fivev = vec_splat( temp_u.v, 0 );
704     temp_u.s[0]=6;
705     sixv = vec_splat( temp_u.v, 0 );
706     temp_u.s[0]=16;
707     sixteenv = (vec_s16_t)vec_splat( temp_u.v, 0 );
708     temp_u.s[0]=32;
709     thirtytwov = (vec_s16_t)vec_splat( temp_u.v, 0 );
710
711     for( int y = 0; y < i_height; y++ )
712     {
713         int x = 0;
714
715         /* horizontal_filter */
716         HPEL_FILTER_HORIZONTAL();
717
718         /* vertical_filter */
719         HPEL_FILTER_VERTICAL();
720
721         /* central_filter */
722         tempav = tempcv;
723         tempbv = tempdv;
724         tempcv = vec_splat( temp1v, 0 ); /* first only */
725         tempdv = temp1v;
726         tempev = temp4v;
727
728         for( x = 16; x < i_width; x+=16 )
729         {
730             /* horizontal_filter */
731             HPEL_FILTER_HORIZONTAL();
732
733             /* vertical_filter */
734             HPEL_FILTER_VERTICAL();
735
736             /* central_filter */
737             tempav = tempcv;
738             tempbv = tempdv;
739             tempcv = tempev;
740             tempdv = temp1v;
741             tempev = temp4v;
742
743             HPEL_FILTER_CENTRAL();
744         }
745
746         /* Partial vertical filter */
747         VEC_LOAD_PARTIAL( &src[x+i_stride*(y-2)], src1v, 16, vec_u8_t, src );
748         VEC_LOAD_PARTIAL( &src[x+i_stride*(y-1)], src2v, 16, vec_u8_t, src );
749         VEC_LOAD_PARTIAL( &src[x+i_stride*(y-0)], src3v, 16, vec_u8_t, src );
750         VEC_LOAD_PARTIAL( &src[x+i_stride*(y+1)], src4v, 16, vec_u8_t, src );
751         VEC_LOAD_PARTIAL( &src[x+i_stride*(y+2)], src5v, 16, vec_u8_t, src );
752         VEC_LOAD_PARTIAL( &src[x+i_stride*(y+3)], src6v, 16, vec_u8_t, src );
753
754         temp1v = vec_u8_to_s16_h( src1v );
755         temp2v = vec_u8_to_s16_h( src2v );
756         temp3v = vec_u8_to_s16_h( src3v );
757         temp4v = vec_u8_to_s16_h( src4v );
758         temp5v = vec_u8_to_s16_h( src5v );
759         temp6v = vec_u8_to_s16_h( src6v );
760
761         HPEL_FILTER_1( temp1v, temp2v, temp3v, temp4v, temp5v, temp6v );
762
763         /* central_filter */
764         tempav = tempcv;
765         tempbv = tempdv;
766         tempcv = tempev;
767         tempdv = temp1v;
768         /* tempev is not used */
769
770         HPEL_FILTER_CENTRAL();
771     }
772 }
773
774 static void frame_init_lowres_core_altivec( uint8_t *src0, uint8_t *dst0, uint8_t *dsth, uint8_t *dstv, uint8_t *dstc,
775                                             intptr_t src_stride, intptr_t dst_stride, int width, int height )
776 {
777     int w = width >> 4;
778     int end = (width & 15);
779     vec_u8_t src0v, src1v, src2v;
780     vec_u8_t lv, hv, src1p1v;
781     vec_u8_t avg0v, avg1v, avghv, avghp1v, avgleftv, avgrightv;
782     static const vec_u8_t inverse_bridge_shuffle = CV(0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x10, 0x12, 0x14, 0x16, 0x18, 0x1A, 0x1C, 0x1E );
783 #ifndef WORDS_BIGENDIAN
784     static const vec_u8_t inverse_bridge_shuffle_1 = CV(0x01, 0x03, 0x05, 0x07, 0x09, 0x0B, 0x0D, 0x0F, 0x11, 0x13, 0x15, 0x17, 0x19, 0x1B, 0x1D, 0x1F );
785 #endif
786
787     for( int y = 0; y < height; y++ )
788     {
789         int x;
790         uint8_t *src1 = src0+src_stride;
791         uint8_t *src2 = src1+src_stride;
792
793         src0v = vec_ld(0, src0);
794         src1v = vec_ld(0, src1);
795         src2v = vec_ld(0, src2);
796
797         avg0v = vec_avg(src0v, src1v);
798         avg1v = vec_avg(src1v, src2v);
799
800         for( x = 0; x < w; x++ )
801         {
802             lv = vec_ld(16*(x*2+1), src0);
803             src1v = vec_ld(16*(x*2+1), src1);
804             avghv = vec_avg(lv, src1v);
805
806             lv = vec_ld(16*(x*2+2), src0);
807             src1p1v = vec_ld(16*(x*2+2), src1);
808             avghp1v = vec_avg(lv, src1p1v);
809
810             avgleftv = vec_avg(VSLD(avg0v, avghv, 1), avg0v);
811             avgrightv = vec_avg(VSLD(avghv, avghp1v, 1), avghv);
812
813             vec_st(vec_perm(avgleftv, avgrightv, inverse_bridge_shuffle), 16*x, dst0);
814 #ifdef WORDS_BIGENDIAN
815             vec_st((vec_u8_t)vec_pack((vec_u16_t)avgleftv,(vec_u16_t)avgrightv), 16*x, dsth);
816 #else
817             vec_st(vec_perm(avgleftv, avgrightv, inverse_bridge_shuffle_1), 16*x, dsth);
818 #endif
819
820             avg0v = avghp1v;
821
822             hv = vec_ld(16*(x*2+1), src2);
823             avghv = vec_avg(src1v, hv);
824
825             hv = vec_ld(16*(x*2+2), src2);
826             avghp1v = vec_avg(src1p1v, hv);
827
828             avgleftv = vec_avg(VSLD(avg1v, avghv, 1), avg1v);
829             avgrightv = vec_avg(VSLD(avghv, avghp1v, 1), avghv);
830
831             vec_st(vec_perm(avgleftv, avgrightv, inverse_bridge_shuffle), 16*x, dstv);
832 #ifdef WORDS_BIGENDIAN
833             vec_st((vec_u8_t)vec_pack((vec_u16_t)avgleftv,(vec_u16_t)avgrightv), 16*x, dstc);
834 #else
835             vec_st(vec_perm(avgleftv, avgrightv, inverse_bridge_shuffle_1), 16*x, dstc);
836 #endif
837
838             avg1v = avghp1v;
839
840         }
841         if( end )
842         {
843             lv = vec_ld(16*(x*2+1), src0);
844             src1v = vec_ld(16*(x*2+1), src1);
845             avghv = vec_avg(lv, src1v);
846
847             lv = vec_ld(16*(x*2+1), src2);
848             avghp1v = vec_avg(src1v, lv);
849
850             avgleftv = vec_avg(VSLD(avg0v, avghv, 1), avg0v);
851             avgrightv = vec_avg(VSLD(avg1v, avghp1v, 1), avg1v);
852
853             lv = vec_perm(avgleftv, avgrightv, inverse_bridge_shuffle);
854 #ifdef WORDS_BIGENDIAN
855             hv = (vec_u8_t)vec_pack((vec_u16_t)avgleftv,(vec_u16_t)avgrightv);
856 #else
857             hv = vec_perm(avgleftv, avgrightv, inverse_bridge_shuffle_1);
858 #endif
859
860             vec_ste((vec_u32_t)lv,16*x,(uint32_t*)dst0);
861             vec_ste((vec_u32_t)lv,16*x+4,(uint32_t*)dst0);
862             vec_ste((vec_u32_t)hv,16*x,(uint32_t*)dsth);
863             vec_ste((vec_u32_t)hv,16*x+4,(uint32_t*)dsth);
864
865             lv = vec_sld(lv, lv, 8);
866             hv = vec_sld(hv, hv, 8);
867
868             vec_ste((vec_u32_t)lv,16*x,(uint32_t*)dstv);
869             vec_ste((vec_u32_t)lv,16*x+4,(uint32_t*)dstv);
870             vec_ste((vec_u32_t)hv,16*x,(uint32_t*)dstc);
871             vec_ste((vec_u32_t)hv,16*x+4,(uint32_t*)dstc);
872         }
873
874         src0 += src_stride*2;
875         dst0 += dst_stride;
876         dsth += dst_stride;
877         dstv += dst_stride;
878         dstc += dst_stride;
879     }
880 }
881
882 static void mc_weight_w2_altivec( uint8_t *dst, intptr_t i_dst, uint8_t *src, intptr_t i_src,
883                                   const x264_weight_t *weight, int i_height )
884 {
885     LOAD_ZERO;
886     PREP_LOAD;
887     PREP_LOAD_SRC( src );
888     vec_u8_t srcv;
889     vec_s16_t weightv;
890     vec_s16_t scalev, offsetv, denomv, roundv;
891     vec_s16_u loadv;
892
893     int denom = weight->i_denom;
894
895     loadv.s[0] = weight->i_scale;
896     scalev = vec_splat( loadv.v, 0 );
897
898     loadv.s[0] = weight->i_offset;
899     offsetv = vec_splat( loadv.v, 0 );
900
901     if( denom >= 1 )
902     {
903         loadv.s[0] = denom;
904         denomv = vec_splat( loadv.v, 0 );
905
906         loadv.s[0] = 1<<(denom - 1);
907         roundv = vec_splat( loadv.v, 0 );
908
909         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
910         {
911             VEC_LOAD( src, srcv, 2, vec_u8_t, src );
912             weightv = vec_u8_to_s16( srcv );
913
914             weightv = vec_mladd( weightv, scalev, roundv );
915             weightv = vec_sra( weightv, (vec_u16_t)denomv );
916             weightv = vec_add( weightv, offsetv );
917
918             srcv = vec_packsu( weightv, zero_s16v );
919             vec_ste( vec_splat( (vec_u16_t)srcv, 0 ), 0, (uint16_t*)dst );
920         }
921     }
922     else
923     {
924         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
925         {
926             VEC_LOAD( src, srcv, 2, vec_u8_t, src );
927             weightv = vec_u8_to_s16( srcv );
928
929             weightv = vec_mladd( weightv, scalev, offsetv );
930
931             srcv = vec_packsu( weightv, zero_s16v );
932             vec_ste( vec_splat( (vec_u16_t)srcv, 0 ), 0, (uint16_t*)dst );
933         }
934     }
935 }
936 static void mc_weight_w4_altivec( uint8_t *dst, intptr_t i_dst, uint8_t *src, intptr_t i_src,
937                                   const x264_weight_t *weight, int i_height )
938 {
939     LOAD_ZERO;
940     PREP_LOAD;
941     PREP_LOAD_SRC( src );
942     vec_u8_t srcv;
943     vec_s16_t weightv;
944     vec_s16_t scalev, offsetv, denomv, roundv;
945     vec_s16_u loadv;
946
947     int denom = weight->i_denom;
948
949     loadv.s[0] = weight->i_scale;
950     scalev = vec_splat( loadv.v, 0 );
951
952     loadv.s[0] = weight->i_offset;
953     offsetv = vec_splat( loadv.v, 0 );
954
955     if( denom >= 1 )
956     {
957         loadv.s[0] = denom;
958         denomv = vec_splat( loadv.v, 0 );
959
960         loadv.s[0] = 1<<(denom - 1);
961         roundv = vec_splat( loadv.v, 0 );
962
963         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
964         {
965             VEC_LOAD( src, srcv, 4, vec_u8_t, src );
966             weightv = vec_u8_to_s16( srcv );
967
968             weightv = vec_mladd( weightv, scalev, roundv );
969             weightv = vec_sra( weightv, (vec_u16_t)denomv );
970             weightv = vec_add( weightv, offsetv );
971
972             srcv = vec_packsu( weightv, zero_s16v );
973             vec_ste( vec_splat( (vec_u32_t)srcv, 0 ), 0, (uint32_t*)dst );
974         }
975     }
976     else
977     {
978         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
979         {
980             VEC_LOAD( src, srcv, 4, vec_u8_t, src );
981             weightv = vec_u8_to_s16( srcv );
982
983             weightv = vec_mladd( weightv, scalev, offsetv );
984
985             srcv = vec_packsu( weightv, zero_s16v );
986             vec_ste( vec_splat( (vec_u32_t)srcv, 0 ), 0, (uint32_t*)dst );
987         }
988     }
989 }
990 static void mc_weight_w8_altivec( uint8_t *dst, intptr_t i_dst, uint8_t *src, intptr_t i_src,
991                                   const x264_weight_t *weight, int i_height )
992 {
993     LOAD_ZERO;
994     PREP_LOAD;
995     PREP_LOAD_SRC( src );
996     PREP_STORE8;
997     vec_u8_t srcv;
998     vec_s16_t weightv;
999     vec_s16_t scalev, offsetv, denomv, roundv;
1000     vec_s16_u loadv;
1001
1002     int denom = weight->i_denom;
1003
1004     loadv.s[0] = weight->i_scale;
1005     scalev = vec_splat( loadv.v, 0 );
1006
1007     loadv.s[0] = weight->i_offset;
1008     offsetv = vec_splat( loadv.v, 0 );
1009
1010     if( denom >= 1 )
1011     {
1012         loadv.s[0] = denom;
1013         denomv = vec_splat( loadv.v, 0 );
1014
1015         loadv.s[0] = 1<<(denom - 1);
1016         roundv = vec_splat( loadv.v, 0 );
1017
1018         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
1019         {
1020             VEC_LOAD( src, srcv, 8, vec_u8_t, src );
1021             weightv = vec_u8_to_s16( srcv );
1022
1023             weightv = vec_mladd( weightv, scalev, roundv );
1024             weightv = vec_sra( weightv, (vec_u16_t)denomv );
1025             weightv = vec_add( weightv, offsetv );
1026
1027             srcv = vec_packsu( weightv, zero_s16v );
1028             VEC_STORE8( srcv, dst );
1029         }
1030     }
1031     else
1032     {
1033         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
1034         {
1035             VEC_LOAD( src, srcv, 8, vec_u8_t, src );
1036             weightv = vec_u8_to_s16( srcv );
1037
1038             weightv = vec_mladd( weightv, scalev, offsetv );
1039
1040             srcv = vec_packsu( weightv, zero_s16v );
1041             VEC_STORE8( srcv, dst );
1042         }
1043     }
1044 }
1045 static void mc_weight_w16_altivec( uint8_t *dst, intptr_t i_dst, uint8_t *src, intptr_t i_src,
1046                                    const x264_weight_t *weight, int i_height )
1047 {
1048     LOAD_ZERO;
1049     PREP_LOAD;
1050     PREP_LOAD_SRC( src );
1051     vec_u8_t srcv;
1052     vec_s16_t weight_lv, weight_hv;
1053     vec_s16_t scalev, offsetv, denomv, roundv;
1054     vec_s16_u loadv;
1055
1056     int denom = weight->i_denom;
1057
1058     loadv.s[0] = weight->i_scale;
1059     scalev = vec_splat( loadv.v, 0 );
1060
1061     loadv.s[0] = weight->i_offset;
1062     offsetv = vec_splat( loadv.v, 0 );
1063
1064     if( denom >= 1 )
1065     {
1066         loadv.s[0] = denom;
1067         denomv = vec_splat( loadv.v, 0 );
1068
1069         loadv.s[0] = 1<<(denom - 1);
1070         roundv = vec_splat( loadv.v, 0 );
1071
1072         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
1073         {
1074             VEC_LOAD( src, srcv, 16, vec_u8_t, src );
1075             weight_hv = vec_u8_to_s16_h( srcv );
1076             weight_lv = vec_u8_to_s16_l( srcv );
1077
1078             weight_hv = vec_mladd( weight_hv, scalev, roundv );
1079             weight_lv = vec_mladd( weight_lv, scalev, roundv );
1080             weight_hv = vec_sra( weight_hv, (vec_u16_t)denomv );
1081             weight_lv = vec_sra( weight_lv, (vec_u16_t)denomv );
1082             weight_hv = vec_add( weight_hv, offsetv );
1083             weight_lv = vec_add( weight_lv, offsetv );
1084
1085             srcv = vec_packsu( weight_hv, weight_lv );
1086             vec_st( srcv, 0, dst );
1087         }
1088     }
1089     else
1090     {
1091         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
1092         {
1093             VEC_LOAD( src, srcv, 16, vec_u8_t, src );
1094             weight_hv = vec_u8_to_s16_h( srcv );
1095             weight_lv = vec_u8_to_s16_l( srcv );
1096
1097             weight_hv = vec_mladd( weight_hv, scalev, offsetv );
1098             weight_lv = vec_mladd( weight_lv, scalev, offsetv );
1099
1100             srcv = vec_packsu( weight_hv, weight_lv );
1101             vec_st( srcv, 0, dst );
1102         }
1103     }
1104 }
1105 static void mc_weight_w20_altivec( uint8_t *dst, intptr_t i_dst, uint8_t *src, intptr_t i_src,
1106                                    const x264_weight_t *weight, int i_height )
1107 {
1108     LOAD_ZERO;
1109     PREP_LOAD_SRC( src );
1110     vec_u8_t src_1v, src_2v, src_3v;
1111     vec_s16_t weight_lv, weight_hv, weight_3v;
1112     vec_s16_t scalev, offsetv, denomv, roundv;
1113     vec_s16_u loadv;
1114
1115     int denom = weight->i_denom;
1116
1117     loadv.s[0] = weight->i_scale;
1118     scalev = vec_splat( loadv.v, 0 );
1119
1120     loadv.s[0] = weight->i_offset;
1121     offsetv = vec_splat( loadv.v, 0 );
1122
1123     if( denom >= 1 )
1124     {
1125         loadv.s[0] = denom;
1126         denomv = vec_splat( loadv.v, 0 );
1127
1128         loadv.s[0] = 1<<(denom - 1);
1129         roundv = vec_splat( loadv.v, 0 );
1130
1131         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
1132         {
1133             src_1v = vec_ld( 0,  src );
1134             src_2v = vec_ld( 16, src );
1135             src_3v = vec_ld( 19, src );
1136             src_1v = vec_perm( src_1v, src_2v, _src_ );
1137             src_3v = vec_perm( src_2v, src_3v, _src_ );
1138             weight_hv = vec_u8_to_s16_h( src_1v );
1139             weight_lv = vec_u8_to_s16_l( src_1v );
1140             weight_3v = vec_u8_to_s16_h( src_3v );
1141
1142             weight_hv = vec_mladd( weight_hv, scalev, roundv );
1143             weight_lv = vec_mladd( weight_lv, scalev, roundv );
1144             weight_3v = vec_mladd( weight_3v, scalev, roundv );
1145             weight_hv = vec_sra( weight_hv, (vec_u16_t)denomv );
1146             weight_lv = vec_sra( weight_lv, (vec_u16_t)denomv );
1147             weight_3v = vec_sra( weight_3v, (vec_u16_t)denomv );
1148             weight_hv = vec_add( weight_hv, offsetv );
1149             weight_lv = vec_add( weight_lv, offsetv );
1150             weight_3v = vec_add( weight_3v, offsetv );
1151
1152             src_1v = vec_packsu( weight_hv, weight_lv );
1153             src_3v = vec_packsu( weight_3v, zero_s16v );
1154             vec_st( src_1v, 0, dst );
1155             vec_ste( (vec_u32_t)src_3v, 16, (uint32_t*)dst );
1156         }
1157     }
1158     else
1159     {
1160         for( int y = 0; y < i_height; y++, dst += i_dst, src += i_src )
1161         {
1162             src_1v = vec_ld( 0,  src );
1163             src_2v = vec_ld( 16, src );
1164             src_3v = vec_ld( 19, src );
1165             src_1v = vec_perm( src_1v, src_2v, _src_ );
1166             src_3v = vec_perm( src_2v, src_3v, _src_ );
1167             weight_hv = vec_u8_to_s16_h( src_1v );
1168             weight_lv = vec_u8_to_s16_l( src_1v );
1169             weight_3v = vec_u8_to_s16_h( src_3v );
1170
1171             weight_hv = vec_mladd( weight_hv, scalev, offsetv );
1172             weight_lv = vec_mladd( weight_lv, scalev, offsetv );
1173             weight_3v = vec_mladd( weight_3v, scalev, offsetv );
1174
1175             src_1v = vec_packsu( weight_hv, weight_lv );
1176             src_3v = vec_packsu( weight_3v, zero_s16v );
1177             vec_st( src_1v, 0, dst );
1178             vec_ste( (vec_u32_t)src_3v, 16, (uint32_t*)dst );
1179         }
1180     }
1181 }
1182
1183 static weight_fn_t x264_mc_weight_wtab_altivec[6] =
1184 {
1185     mc_weight_w2_altivec,
1186     mc_weight_w4_altivec,
1187     mc_weight_w8_altivec,
1188     mc_weight_w16_altivec,
1189     mc_weight_w16_altivec,
1190     mc_weight_w20_altivec,
1191 };
1192
1193 #endif // !HIGH_BIT_DEPTH
1194
1195 void x264_mc_altivec_init( x264_mc_functions_t *pf )
1196 {
1197 #if !HIGH_BIT_DEPTH
1198     pf->mc_luma   = mc_luma_altivec;
1199     pf->get_ref   = get_ref_altivec;
1200     pf->mc_chroma = mc_chroma_altivec;
1201
1202     pf->copy_16x16_unaligned = x264_mc_copy_w16_altivec;
1203     pf->copy[PIXEL_16x16] = x264_mc_copy_w16_aligned_altivec;
1204
1205     pf->hpel_filter = x264_hpel_filter_altivec;
1206     pf->frame_init_lowres_core = frame_init_lowres_core_altivec;
1207
1208     pf->weight = x264_mc_weight_wtab_altivec;
1209 #endif // !HIGH_BIT_DEPTH
1210 }