dsputil: Propagate bit depth information to all (sub)init functions
[ffmpeg.git] / libavcodec / dsputil.c
1 /*
2  * DSP utils
3  * Copyright (c) 2000, 2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * gmc & q-pel & 32/64 bit based MC by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of Libav.
9  *
10  * Libav is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * Libav is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with Libav; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * DSP utils
28  */
29
30 #include "libavutil/attributes.h"
31 #include "libavutil/imgutils.h"
32 #include "avcodec.h"
33 #include "copy_block.h"
34 #include "dct.h"
35 #include "dsputil.h"
36 #include "simple_idct.h"
37 #include "faandct.h"
38 #include "faanidct.h"
39 #include "imgconvert.h"
40 #include "mathops.h"
41 #include "mpegvideo.h"
42 #include "config.h"
43
44 uint32_t ff_square_tab[512] = { 0, };
45
46 #define BIT_DEPTH 16
47 #include "dsputil_template.c"
48 #undef BIT_DEPTH
49
50 #define BIT_DEPTH 8
51 #include "dsputil_template.c"
52
53 // 0x7f7f7f7f or 0x7f7f7f7f7f7f7f7f or whatever, depending on the cpu's native arithmetic size
54 #define pb_7f (~0UL / 255 * 0x7f)
55 #define pb_80 (~0UL / 255 * 0x80)
56
57 /* Specific zigzag scan for 248 idct. NOTE that unlike the
58  * specification, we interleave the fields */
59 const uint8_t ff_zigzag248_direct[64] = {
60      0,  8,  1,  9, 16, 24,  2, 10,
61     17, 25, 32, 40, 48, 56, 33, 41,
62     18, 26,  3, 11,  4, 12, 19, 27,
63     34, 42, 49, 57, 50, 58, 35, 43,
64     20, 28,  5, 13,  6, 14, 21, 29,
65     36, 44, 51, 59, 52, 60, 37, 45,
66     22, 30,  7, 15, 23, 31, 38, 46,
67     53, 61, 54, 62, 39, 47, 55, 63,
68 };
69
70 const uint8_t ff_alternate_horizontal_scan[64] = {
71      0,  1,  2,  3,  8,  9, 16, 17,
72     10, 11,  4,  5,  6,  7, 15, 14,
73     13, 12, 19, 18, 24, 25, 32, 33,
74     26, 27, 20, 21, 22, 23, 28, 29,
75     30, 31, 34, 35, 40, 41, 48, 49,
76     42, 43, 36, 37, 38, 39, 44, 45,
77     46, 47, 50, 51, 56, 57, 58, 59,
78     52, 53, 54, 55, 60, 61, 62, 63,
79 };
80
81 const uint8_t ff_alternate_vertical_scan[64] = {
82      0,  8, 16, 24,  1,  9,  2, 10,
83     17, 25, 32, 40, 48, 56, 57, 49,
84     41, 33, 26, 18,  3, 11,  4, 12,
85     19, 27, 34, 42, 50, 58, 35, 43,
86     51, 59, 20, 28,  5, 13,  6, 14,
87     21, 29, 36, 44, 52, 60, 37, 45,
88     53, 61, 22, 30,  7, 15, 23, 31,
89     38, 46, 54, 62, 39, 47, 55, 63,
90 };
91
92 /* Input permutation for the simple_idct_mmx */
93 static const uint8_t simple_mmx_permutation[64] = {
94     0x00, 0x08, 0x04, 0x09, 0x01, 0x0C, 0x05, 0x0D,
95     0x10, 0x18, 0x14, 0x19, 0x11, 0x1C, 0x15, 0x1D,
96     0x20, 0x28, 0x24, 0x29, 0x21, 0x2C, 0x25, 0x2D,
97     0x12, 0x1A, 0x16, 0x1B, 0x13, 0x1E, 0x17, 0x1F,
98     0x02, 0x0A, 0x06, 0x0B, 0x03, 0x0E, 0x07, 0x0F,
99     0x30, 0x38, 0x34, 0x39, 0x31, 0x3C, 0x35, 0x3D,
100     0x22, 0x2A, 0x26, 0x2B, 0x23, 0x2E, 0x27, 0x2F,
101     0x32, 0x3A, 0x36, 0x3B, 0x33, 0x3E, 0x37, 0x3F,
102 };
103
104 static const uint8_t idct_sse2_row_perm[8] = { 0, 4, 1, 5, 2, 6, 3, 7 };
105
106 av_cold void ff_init_scantable(uint8_t *permutation, ScanTable *st,
107                                const uint8_t *src_scantable)
108 {
109     int i, end;
110
111     st->scantable = src_scantable;
112
113     for (i = 0; i < 64; i++) {
114         int j = src_scantable[i];
115         st->permutated[i] = permutation[j];
116     }
117
118     end = -1;
119     for (i = 0; i < 64; i++) {
120         int j = st->permutated[i];
121         if (j > end)
122             end = j;
123         st->raster_end[i] = end;
124     }
125 }
126
127 av_cold void ff_init_scantable_permutation(uint8_t *idct_permutation,
128                                            int idct_permutation_type)
129 {
130     int i;
131
132     switch (idct_permutation_type) {
133     case FF_NO_IDCT_PERM:
134         for (i = 0; i < 64; i++)
135             idct_permutation[i] = i;
136         break;
137     case FF_LIBMPEG2_IDCT_PERM:
138         for (i = 0; i < 64; i++)
139             idct_permutation[i] = (i & 0x38) | ((i & 6) >> 1) | ((i & 1) << 2);
140         break;
141     case FF_SIMPLE_IDCT_PERM:
142         for (i = 0; i < 64; i++)
143             idct_permutation[i] = simple_mmx_permutation[i];
144         break;
145     case FF_TRANSPOSE_IDCT_PERM:
146         for (i = 0; i < 64; i++)
147             idct_permutation[i] = ((i & 7) << 3) | (i >> 3);
148         break;
149     case FF_PARTTRANS_IDCT_PERM:
150         for (i = 0; i < 64; i++)
151             idct_permutation[i] = (i & 0x24) | ((i & 3) << 3) | ((i >> 3) & 3);
152         break;
153     case FF_SSE2_IDCT_PERM:
154         for (i = 0; i < 64; i++)
155             idct_permutation[i] = (i & 0x38) | idct_sse2_row_perm[i & 7];
156         break;
157     default:
158         av_log(NULL, AV_LOG_ERROR,
159                "Internal error, IDCT permutation not set\n");
160     }
161 }
162
163 static int pix_sum_c(uint8_t *pix, int line_size)
164 {
165     int s = 0, i, j;
166
167     for (i = 0; i < 16; i++) {
168         for (j = 0; j < 16; j += 8) {
169             s   += pix[0];
170             s   += pix[1];
171             s   += pix[2];
172             s   += pix[3];
173             s   += pix[4];
174             s   += pix[5];
175             s   += pix[6];
176             s   += pix[7];
177             pix += 8;
178         }
179         pix += line_size - 16;
180     }
181     return s;
182 }
183
184 static int pix_norm1_c(uint8_t *pix, int line_size)
185 {
186     int s = 0, i, j;
187     uint32_t *sq = ff_square_tab + 256;
188
189     for (i = 0; i < 16; i++) {
190         for (j = 0; j < 16; j += 8) {
191 #if 0
192             s += sq[pix[0]];
193             s += sq[pix[1]];
194             s += sq[pix[2]];
195             s += sq[pix[3]];
196             s += sq[pix[4]];
197             s += sq[pix[5]];
198             s += sq[pix[6]];
199             s += sq[pix[7]];
200 #else
201 #if HAVE_FAST_64BIT
202             register uint64_t x = *(uint64_t *) pix;
203             s += sq[x         & 0xff];
204             s += sq[(x >>  8) & 0xff];
205             s += sq[(x >> 16) & 0xff];
206             s += sq[(x >> 24) & 0xff];
207             s += sq[(x >> 32) & 0xff];
208             s += sq[(x >> 40) & 0xff];
209             s += sq[(x >> 48) & 0xff];
210             s += sq[(x >> 56) & 0xff];
211 #else
212             register uint32_t x = *(uint32_t *) pix;
213             s += sq[x         & 0xff];
214             s += sq[(x >>  8) & 0xff];
215             s += sq[(x >> 16) & 0xff];
216             s += sq[(x >> 24) & 0xff];
217             x  = *(uint32_t *) (pix + 4);
218             s += sq[x         & 0xff];
219             s += sq[(x >>  8) & 0xff];
220             s += sq[(x >> 16) & 0xff];
221             s += sq[(x >> 24) & 0xff];
222 #endif
223 #endif
224             pix += 8;
225         }
226         pix += line_size - 16;
227     }
228     return s;
229 }
230
231 static void bswap_buf(uint32_t *dst, const uint32_t *src, int w)
232 {
233     int i;
234
235     for (i = 0; i + 8 <= w; i += 8) {
236         dst[i + 0] = av_bswap32(src[i + 0]);
237         dst[i + 1] = av_bswap32(src[i + 1]);
238         dst[i + 2] = av_bswap32(src[i + 2]);
239         dst[i + 3] = av_bswap32(src[i + 3]);
240         dst[i + 4] = av_bswap32(src[i + 4]);
241         dst[i + 5] = av_bswap32(src[i + 5]);
242         dst[i + 6] = av_bswap32(src[i + 6]);
243         dst[i + 7] = av_bswap32(src[i + 7]);
244     }
245     for (; i < w; i++)
246         dst[i + 0] = av_bswap32(src[i + 0]);
247 }
248
249 static void bswap16_buf(uint16_t *dst, const uint16_t *src, int len)
250 {
251     while (len--)
252         *dst++ = av_bswap16(*src++);
253 }
254
255 static int sse4_c(void *v, uint8_t *pix1, uint8_t *pix2, int line_size, int h)
256 {
257     int s = 0, i;
258     uint32_t *sq = ff_square_tab + 256;
259
260     for (i = 0; i < h; i++) {
261         s    += sq[pix1[0] - pix2[0]];
262         s    += sq[pix1[1] - pix2[1]];
263         s    += sq[pix1[2] - pix2[2]];
264         s    += sq[pix1[3] - pix2[3]];
265         pix1 += line_size;
266         pix2 += line_size;
267     }
268     return s;
269 }
270
271 static int sse8_c(void *v, uint8_t *pix1, uint8_t *pix2, int line_size, int h)
272 {
273     int s = 0, i;
274     uint32_t *sq = ff_square_tab + 256;
275
276     for (i = 0; i < h; i++) {
277         s    += sq[pix1[0] - pix2[0]];
278         s    += sq[pix1[1] - pix2[1]];
279         s    += sq[pix1[2] - pix2[2]];
280         s    += sq[pix1[3] - pix2[3]];
281         s    += sq[pix1[4] - pix2[4]];
282         s    += sq[pix1[5] - pix2[5]];
283         s    += sq[pix1[6] - pix2[6]];
284         s    += sq[pix1[7] - pix2[7]];
285         pix1 += line_size;
286         pix2 += line_size;
287     }
288     return s;
289 }
290
291 static int sse16_c(void *v, uint8_t *pix1, uint8_t *pix2, int line_size, int h)
292 {
293     int s = 0, i;
294     uint32_t *sq = ff_square_tab + 256;
295
296     for (i = 0; i < h; i++) {
297         s += sq[pix1[0]  - pix2[0]];
298         s += sq[pix1[1]  - pix2[1]];
299         s += sq[pix1[2]  - pix2[2]];
300         s += sq[pix1[3]  - pix2[3]];
301         s += sq[pix1[4]  - pix2[4]];
302         s += sq[pix1[5]  - pix2[5]];
303         s += sq[pix1[6]  - pix2[6]];
304         s += sq[pix1[7]  - pix2[7]];
305         s += sq[pix1[8]  - pix2[8]];
306         s += sq[pix1[9]  - pix2[9]];
307         s += sq[pix1[10] - pix2[10]];
308         s += sq[pix1[11] - pix2[11]];
309         s += sq[pix1[12] - pix2[12]];
310         s += sq[pix1[13] - pix2[13]];
311         s += sq[pix1[14] - pix2[14]];
312         s += sq[pix1[15] - pix2[15]];
313
314         pix1 += line_size;
315         pix2 += line_size;
316     }
317     return s;
318 }
319
320 static void diff_pixels_c(int16_t *restrict block, const uint8_t *s1,
321                           const uint8_t *s2, int stride)
322 {
323     int i;
324
325     /* read the pixels */
326     for (i = 0; i < 8; i++) {
327         block[0] = s1[0] - s2[0];
328         block[1] = s1[1] - s2[1];
329         block[2] = s1[2] - s2[2];
330         block[3] = s1[3] - s2[3];
331         block[4] = s1[4] - s2[4];
332         block[5] = s1[5] - s2[5];
333         block[6] = s1[6] - s2[6];
334         block[7] = s1[7] - s2[7];
335         s1      += stride;
336         s2      += stride;
337         block   += 8;
338     }
339 }
340
341 static void put_pixels_clamped_c(const int16_t *block, uint8_t *restrict pixels,
342                                  int line_size)
343 {
344     int i;
345
346     /* read the pixels */
347     for (i = 0; i < 8; i++) {
348         pixels[0] = av_clip_uint8(block[0]);
349         pixels[1] = av_clip_uint8(block[1]);
350         pixels[2] = av_clip_uint8(block[2]);
351         pixels[3] = av_clip_uint8(block[3]);
352         pixels[4] = av_clip_uint8(block[4]);
353         pixels[5] = av_clip_uint8(block[5]);
354         pixels[6] = av_clip_uint8(block[6]);
355         pixels[7] = av_clip_uint8(block[7]);
356
357         pixels += line_size;
358         block  += 8;
359     }
360 }
361
362 static void put_signed_pixels_clamped_c(const int16_t *block,
363                                         uint8_t *restrict pixels,
364                                         int line_size)
365 {
366     int i, j;
367
368     for (i = 0; i < 8; i++) {
369         for (j = 0; j < 8; j++) {
370             if (*block < -128)
371                 *pixels = 0;
372             else if (*block > 127)
373                 *pixels = 255;
374             else
375                 *pixels = (uint8_t) (*block + 128);
376             block++;
377             pixels++;
378         }
379         pixels += (line_size - 8);
380     }
381 }
382
383 static void add_pixels8_c(uint8_t *restrict pixels, int16_t *block,
384                           int line_size)
385 {
386     int i;
387
388     for (i = 0; i < 8; i++) {
389         pixels[0] += block[0];
390         pixels[1] += block[1];
391         pixels[2] += block[2];
392         pixels[3] += block[3];
393         pixels[4] += block[4];
394         pixels[5] += block[5];
395         pixels[6] += block[6];
396         pixels[7] += block[7];
397         pixels    += line_size;
398         block     += 8;
399     }
400 }
401
402 static void add_pixels_clamped_c(const int16_t *block, uint8_t *restrict pixels,
403                                  int line_size)
404 {
405     int i;
406
407     /* read the pixels */
408     for (i = 0; i < 8; i++) {
409         pixels[0] = av_clip_uint8(pixels[0] + block[0]);
410         pixels[1] = av_clip_uint8(pixels[1] + block[1]);
411         pixels[2] = av_clip_uint8(pixels[2] + block[2]);
412         pixels[3] = av_clip_uint8(pixels[3] + block[3]);
413         pixels[4] = av_clip_uint8(pixels[4] + block[4]);
414         pixels[5] = av_clip_uint8(pixels[5] + block[5]);
415         pixels[6] = av_clip_uint8(pixels[6] + block[6]);
416         pixels[7] = av_clip_uint8(pixels[7] + block[7]);
417         pixels   += line_size;
418         block    += 8;
419     }
420 }
421
422 static int sum_abs_dctelem_c(int16_t *block)
423 {
424     int sum = 0, i;
425
426     for (i = 0; i < 64; i++)
427         sum += FFABS(block[i]);
428     return sum;
429 }
430
431 static void fill_block16_c(uint8_t *block, uint8_t value, int line_size, int h)
432 {
433     int i;
434
435     for (i = 0; i < h; i++) {
436         memset(block, value, 16);
437         block += line_size;
438     }
439 }
440
441 static void fill_block8_c(uint8_t *block, uint8_t value, int line_size, int h)
442 {
443     int i;
444
445     for (i = 0; i < h; i++) {
446         memset(block, value, 8);
447         block += line_size;
448     }
449 }
450
451 #define avg2(a, b) ((a + b + 1) >> 1)
452 #define avg4(a, b, c, d) ((a + b + c + d + 2) >> 2)
453
454 static void gmc1_c(uint8_t *dst, uint8_t *src, int stride, int h,
455                    int x16, int y16, int rounder)
456 {
457     const int A = (16 - x16) * (16 - y16);
458     const int B = (x16)      * (16 - y16);
459     const int C = (16 - x16) * (y16);
460     const int D = (x16)      * (y16);
461     int i;
462
463     for (i = 0; i < h; i++) {
464         dst[0] = (A * src[0] + B * src[1] + C * src[stride + 0] + D * src[stride + 1] + rounder) >> 8;
465         dst[1] = (A * src[1] + B * src[2] + C * src[stride + 1] + D * src[stride + 2] + rounder) >> 8;
466         dst[2] = (A * src[2] + B * src[3] + C * src[stride + 2] + D * src[stride + 3] + rounder) >> 8;
467         dst[3] = (A * src[3] + B * src[4] + C * src[stride + 3] + D * src[stride + 4] + rounder) >> 8;
468         dst[4] = (A * src[4] + B * src[5] + C * src[stride + 4] + D * src[stride + 5] + rounder) >> 8;
469         dst[5] = (A * src[5] + B * src[6] + C * src[stride + 5] + D * src[stride + 6] + rounder) >> 8;
470         dst[6] = (A * src[6] + B * src[7] + C * src[stride + 6] + D * src[stride + 7] + rounder) >> 8;
471         dst[7] = (A * src[7] + B * src[8] + C * src[stride + 7] + D * src[stride + 8] + rounder) >> 8;
472         dst   += stride;
473         src   += stride;
474     }
475 }
476
477 void ff_gmc_c(uint8_t *dst, uint8_t *src, int stride, int h, int ox, int oy,
478               int dxx, int dxy, int dyx, int dyy, int shift, int r,
479               int width, int height)
480 {
481     int y, vx, vy;
482     const int s = 1 << shift;
483
484     width--;
485     height--;
486
487     for (y = 0; y < h; y++) {
488         int x;
489
490         vx = ox;
491         vy = oy;
492         for (x = 0; x < 8; x++) { // FIXME: optimize
493             int index;
494             int src_x  = vx >> 16;
495             int src_y  = vy >> 16;
496             int frac_x = src_x & (s - 1);
497             int frac_y = src_y & (s - 1);
498
499             src_x >>= shift;
500             src_y >>= shift;
501
502             if ((unsigned) src_x < width) {
503                 if ((unsigned) src_y < height) {
504                     index = src_x + src_y * stride;
505                     dst[y * stride + x] =
506                         ((src[index]                        * (s - frac_x) +
507                           src[index + 1]          * frac_x) * (s - frac_y) +
508                          (src[index + stride]               * (s - frac_x) +
509                           src[index + stride + 1] * frac_x) *      frac_y  +
510                          r) >> (shift * 2);
511                 } else {
512                     index = src_x + av_clip(src_y, 0, height) * stride;
513                     dst[y * stride + x] =
514                         ((src[index]               * (s - frac_x) +
515                           src[index + 1] * frac_x) *  s           +
516                          r) >> (shift * 2);
517                 }
518             } else {
519                 if ((unsigned) src_y < height) {
520                     index = av_clip(src_x, 0, width) + src_y * stride;
521                     dst[y * stride + x] =
522                         ((src[index]                    * (s - frac_y) +
523                           src[index + stride] * frac_y) *  s           +
524                          r) >> (shift * 2);
525                 } else {
526                     index = av_clip(src_x, 0, width) +
527                             av_clip(src_y, 0, height) * stride;
528                     dst[y * stride + x] = src[index];
529                 }
530             }
531
532             vx += dxx;
533             vy += dyx;
534         }
535         ox += dxy;
536         oy += dyy;
537     }
538 }
539
540 static inline void put_tpel_pixels_mc00_c(uint8_t *dst, const uint8_t *src,
541                                           int stride, int width, int height)
542 {
543     switch (width) {
544     case 2:
545         put_pixels2_8_c(dst, src, stride, height);
546         break;
547     case 4:
548         put_pixels4_8_c(dst, src, stride, height);
549         break;
550     case 8:
551         put_pixels8_8_c(dst, src, stride, height);
552         break;
553     case 16:
554         put_pixels16_8_c(dst, src, stride, height);
555         break;
556     }
557 }
558
559 static inline void put_tpel_pixels_mc10_c(uint8_t *dst, const uint8_t *src,
560                                           int stride, int width, int height)
561 {
562     int i, j;
563
564     for (i = 0; i < height; i++) {
565         for (j = 0; j < width; j++)
566             dst[j] = ((2 * src[j] + src[j + 1] + 1) *
567                       683) >> 11;
568         src += stride;
569         dst += stride;
570     }
571 }
572
573 static inline void put_tpel_pixels_mc20_c(uint8_t *dst, const uint8_t *src,
574                                           int stride, int width, int height)
575 {
576     int i, j;
577
578     for (i = 0; i < height; i++) {
579         for (j = 0; j < width; j++)
580             dst[j] = ((src[j] + 2 * src[j + 1] + 1) *
581                       683) >> 11;
582         src += stride;
583         dst += stride;
584     }
585 }
586
587 static inline void put_tpel_pixels_mc01_c(uint8_t *dst, const uint8_t *src,
588                                           int stride, int width, int height)
589 {
590     int i, j;
591
592     for (i = 0; i < height; i++) {
593         for (j = 0; j < width; j++)
594             dst[j] = ((2 * src[j] + src[j + stride] + 1) *
595                       683) >> 11;
596         src += stride;
597         dst += stride;
598     }
599 }
600
601 static inline void put_tpel_pixels_mc11_c(uint8_t *dst, const uint8_t *src,
602                                           int stride, int width, int height)
603 {
604     int i, j;
605
606     for (i = 0; i < height; i++) {
607         for (j = 0; j < width; j++)
608             dst[j] = ((4 * src[j]          + 3 * src[j + 1] +
609                        3 * src[j + stride] + 2 * src[j + stride + 1] + 6) *
610                       2731) >> 15;
611         src += stride;
612         dst += stride;
613     }
614 }
615
616 static inline void put_tpel_pixels_mc12_c(uint8_t *dst, const uint8_t *src,
617                                           int stride, int width, int height)
618 {
619     int i, j;
620
621     for (i = 0; i < height; i++) {
622         for (j = 0; j < width; j++)
623             dst[j] = ((3 * src[j]          + 2 * src[j + 1] +
624                        4 * src[j + stride] + 3 * src[j + stride + 1] + 6) *
625                       2731) >> 15;
626         src += stride;
627         dst += stride;
628     }
629 }
630
631 static inline void put_tpel_pixels_mc02_c(uint8_t *dst, const uint8_t *src,
632                                           int stride, int width, int height)
633 {
634     int i, j;
635
636     for (i = 0; i < height; i++) {
637         for (j = 0; j < width; j++)
638             dst[j] = ((src[j] + 2 * src[j + stride] + 1) *
639                       683) >> 11;
640         src += stride;
641         dst += stride;
642     }
643 }
644
645 static inline void put_tpel_pixels_mc21_c(uint8_t *dst, const uint8_t *src,
646                                           int stride, int width, int height)
647 {
648     int i, j;
649
650     for (i = 0; i < height; i++) {
651         for (j = 0; j < width; j++)
652             dst[j] = ((3 * src[j]          + 4 * src[j + 1] +
653                        2 * src[j + stride] + 3 * src[j + stride + 1] + 6) *
654                       2731) >> 15;
655         src += stride;
656         dst += stride;
657     }
658 }
659
660 static inline void put_tpel_pixels_mc22_c(uint8_t *dst, const uint8_t *src,
661                                           int stride, int width, int height)
662 {
663     int i, j;
664
665     for (i = 0; i < height; i++) {
666         for (j = 0; j < width; j++)
667             dst[j] = ((2 * src[j]          + 3 * src[j + 1] +
668                        3 * src[j + stride] + 4 * src[j + stride + 1] + 6) *
669                       2731) >> 15;
670         src += stride;
671         dst += stride;
672     }
673 }
674
675 static inline void avg_tpel_pixels_mc00_c(uint8_t *dst, const uint8_t *src,
676                                           int stride, int width, int height)
677 {
678     switch (width) {
679     case 2:
680         avg_pixels2_8_c(dst, src, stride, height);
681         break;
682     case 4:
683         avg_pixels4_8_c(dst, src, stride, height);
684         break;
685     case 8:
686         avg_pixels8_8_c(dst, src, stride, height);
687         break;
688     case 16:
689         avg_pixels16_8_c(dst, src, stride, height);
690         break;
691     }
692 }
693
694 static inline void avg_tpel_pixels_mc10_c(uint8_t *dst, const uint8_t *src,
695                                           int stride, int width, int height)
696 {
697     int i, j;
698
699     for (i = 0; i < height; i++) {
700         for (j = 0; j < width; j++)
701             dst[j] = (dst[j] +
702                       (((2 * src[j] + src[j + 1] + 1) *
703                         683) >> 11) + 1) >> 1;
704         src += stride;
705         dst += stride;
706     }
707 }
708
709 static inline void avg_tpel_pixels_mc20_c(uint8_t *dst, const uint8_t *src,
710                                           int stride, int width, int height)
711 {
712     int i, j;
713
714     for (i = 0; i < height; i++) {
715         for (j = 0; j < width; j++)
716             dst[j] = (dst[j] +
717                       (((src[j] + 2 * src[j + 1] + 1) *
718                         683) >> 11) + 1) >> 1;
719         src += stride;
720         dst += stride;
721     }
722 }
723
724 static inline void avg_tpel_pixels_mc01_c(uint8_t *dst, const uint8_t *src,
725                                           int stride, int width, int height)
726 {
727     int i, j;
728
729     for (i = 0; i < height; i++) {
730         for (j = 0; j < width; j++)
731             dst[j] = (dst[j] +
732                       (((2 * src[j] + src[j + stride] + 1) *
733                         683) >> 11) + 1) >> 1;
734         src += stride;
735         dst += stride;
736     }
737 }
738
739 static inline void avg_tpel_pixels_mc11_c(uint8_t *dst, const uint8_t *src,
740                                           int stride, int width, int height)
741 {
742     int i, j;
743
744     for (i = 0; i < height; i++) {
745         for (j = 0; j < width; j++)
746             dst[j] = (dst[j] +
747                       (((4 * src[j]          + 3 * src[j + 1] +
748                          3 * src[j + stride] + 2 * src[j + stride + 1] + 6) *
749                         2731) >> 15) + 1) >> 1;
750         src += stride;
751         dst += stride;
752     }
753 }
754
755 static inline void avg_tpel_pixels_mc12_c(uint8_t *dst, const uint8_t *src,
756                                           int stride, int width, int height)
757 {
758     int i, j;
759
760     for (i = 0; i < height; i++) {
761         for (j = 0; j < width; j++)
762             dst[j] = (dst[j] +
763                       (((3 * src[j]          + 2 * src[j + 1] +
764                          4 * src[j + stride] + 3 * src[j + stride + 1] + 6) *
765                         2731) >> 15) + 1) >> 1;
766         src += stride;
767         dst += stride;
768     }
769 }
770
771 static inline void avg_tpel_pixels_mc02_c(uint8_t *dst, const uint8_t *src,
772                                           int stride, int width, int height)
773 {
774     int i, j;
775
776     for (i = 0; i < height; i++) {
777         for (j = 0; j < width; j++)
778             dst[j] = (dst[j] +
779                       (((src[j] + 2 * src[j + stride] + 1) *
780                         683) >> 11) + 1) >> 1;
781         src += stride;
782         dst += stride;
783     }
784 }
785
786 static inline void avg_tpel_pixels_mc21_c(uint8_t *dst, const uint8_t *src,
787                                           int stride, int width, int height)
788 {
789     int i, j;
790
791     for (i = 0; i < height; i++) {
792         for (j = 0; j < width; j++)
793             dst[j] = (dst[j] +
794                       (((3 * src[j]          + 4 * src[j + 1] +
795                          2 * src[j + stride] + 3 * src[j + stride + 1] + 6) *
796                         2731) >> 15) + 1) >> 1;
797         src += stride;
798         dst += stride;
799     }
800 }
801
802 static inline void avg_tpel_pixels_mc22_c(uint8_t *dst, const uint8_t *src,
803                                           int stride, int width, int height)
804 {
805     int i, j;
806
807     for (i = 0; i < height; i++) {
808         for (j = 0; j < width; j++)
809             dst[j] = (dst[j] +
810                       (((2 * src[j]          + 3 * src[j + 1] +
811                          3 * src[j + stride] + 4 * src[j + stride + 1] + 6) *
812                         2731) >> 15) + 1) >> 1;
813         src += stride;
814         dst += stride;
815     }
816 }
817
818 #define QPEL_MC(r, OPNAME, RND, OP)                                           \
819 static void OPNAME ## mpeg4_qpel8_h_lowpass(uint8_t *dst, uint8_t *src,       \
820                                             int dstStride, int srcStride,     \
821                                             int h)                            \
822 {                                                                             \
823     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;                           \
824     int i;                                                                    \
825                                                                               \
826     for (i = 0; i < h; i++) {                                                 \
827         OP(dst[0], (src[0] + src[1]) * 20 - (src[0] + src[2]) * 6 + (src[1] + src[3]) * 3 - (src[2] + src[4])); \
828         OP(dst[1], (src[1] + src[2]) * 20 - (src[0] + src[3]) * 6 + (src[0] + src[4]) * 3 - (src[1] + src[5])); \
829         OP(dst[2], (src[2] + src[3]) * 20 - (src[1] + src[4]) * 6 + (src[0] + src[5]) * 3 - (src[0] + src[6])); \
830         OP(dst[3], (src[3] + src[4]) * 20 - (src[2] + src[5]) * 6 + (src[1] + src[6]) * 3 - (src[0] + src[7])); \
831         OP(dst[4], (src[4] + src[5]) * 20 - (src[3] + src[6]) * 6 + (src[2] + src[7]) * 3 - (src[1] + src[8])); \
832         OP(dst[5], (src[5] + src[6]) * 20 - (src[4] + src[7]) * 6 + (src[3] + src[8]) * 3 - (src[2] + src[8])); \
833         OP(dst[6], (src[6] + src[7]) * 20 - (src[5] + src[8]) * 6 + (src[4] + src[8]) * 3 - (src[3] + src[7])); \
834         OP(dst[7], (src[7] + src[8]) * 20 - (src[6] + src[8]) * 6 + (src[5] + src[7]) * 3 - (src[4] + src[6])); \
835         dst += dstStride;                                                     \
836         src += srcStride;                                                     \
837     }                                                                         \
838 }                                                                             \
839                                                                               \
840 static void OPNAME ## mpeg4_qpel8_v_lowpass(uint8_t *dst, uint8_t *src,       \
841                                             int dstStride, int srcStride)     \
842 {                                                                             \
843     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;                           \
844     const int w = 8;                                                          \
845     int i;                                                                    \
846                                                                               \
847     for (i = 0; i < w; i++) {                                                 \
848         const int src0 = src[0 * srcStride];                                  \
849         const int src1 = src[1 * srcStride];                                  \
850         const int src2 = src[2 * srcStride];                                  \
851         const int src3 = src[3 * srcStride];                                  \
852         const int src4 = src[4 * srcStride];                                  \
853         const int src5 = src[5 * srcStride];                                  \
854         const int src6 = src[6 * srcStride];                                  \
855         const int src7 = src[7 * srcStride];                                  \
856         const int src8 = src[8 * srcStride];                                  \
857         OP(dst[0 * dstStride], (src0 + src1) * 20 - (src0 + src2) * 6 + (src1 + src3) * 3 - (src2 + src4)); \
858         OP(dst[1 * dstStride], (src1 + src2) * 20 - (src0 + src3) * 6 + (src0 + src4) * 3 - (src1 + src5)); \
859         OP(dst[2 * dstStride], (src2 + src3) * 20 - (src1 + src4) * 6 + (src0 + src5) * 3 - (src0 + src6)); \
860         OP(dst[3 * dstStride], (src3 + src4) * 20 - (src2 + src5) * 6 + (src1 + src6) * 3 - (src0 + src7)); \
861         OP(dst[4 * dstStride], (src4 + src5) * 20 - (src3 + src6) * 6 + (src2 + src7) * 3 - (src1 + src8)); \
862         OP(dst[5 * dstStride], (src5 + src6) * 20 - (src4 + src7) * 6 + (src3 + src8) * 3 - (src2 + src8)); \
863         OP(dst[6 * dstStride], (src6 + src7) * 20 - (src5 + src8) * 6 + (src4 + src8) * 3 - (src3 + src7)); \
864         OP(dst[7 * dstStride], (src7 + src8) * 20 - (src6 + src8) * 6 + (src5 + src7) * 3 - (src4 + src6)); \
865         dst++;                                                                \
866         src++;                                                                \
867     }                                                                         \
868 }                                                                             \
869                                                                               \
870 static void OPNAME ## mpeg4_qpel16_h_lowpass(uint8_t *dst, uint8_t *src,      \
871                                              int dstStride, int srcStride,    \
872                                              int h)                           \
873 {                                                                             \
874     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;                           \
875     int i;                                                                    \
876                                                                               \
877     for (i = 0; i < h; i++) {                                                 \
878         OP(dst[0],  (src[0]  + src[1])  * 20 - (src[0]  + src[2])  * 6 + (src[1]  + src[3])  * 3 - (src[2]  + src[4]));  \
879         OP(dst[1],  (src[1]  + src[2])  * 20 - (src[0]  + src[3])  * 6 + (src[0]  + src[4])  * 3 - (src[1]  + src[5]));  \
880         OP(dst[2],  (src[2]  + src[3])  * 20 - (src[1]  + src[4])  * 6 + (src[0]  + src[5])  * 3 - (src[0]  + src[6]));  \
881         OP(dst[3],  (src[3]  + src[4])  * 20 - (src[2]  + src[5])  * 6 + (src[1]  + src[6])  * 3 - (src[0]  + src[7]));  \
882         OP(dst[4],  (src[4]  + src[5])  * 20 - (src[3]  + src[6])  * 6 + (src[2]  + src[7])  * 3 - (src[1]  + src[8]));  \
883         OP(dst[5],  (src[5]  + src[6])  * 20 - (src[4]  + src[7])  * 6 + (src[3]  + src[8])  * 3 - (src[2]  + src[9]));  \
884         OP(dst[6],  (src[6]  + src[7])  * 20 - (src[5]  + src[8])  * 6 + (src[4]  + src[9])  * 3 - (src[3]  + src[10])); \
885         OP(dst[7],  (src[7]  + src[8])  * 20 - (src[6]  + src[9])  * 6 + (src[5]  + src[10]) * 3 - (src[4]  + src[11])); \
886         OP(dst[8],  (src[8]  + src[9])  * 20 - (src[7]  + src[10]) * 6 + (src[6]  + src[11]) * 3 - (src[5]  + src[12])); \
887         OP(dst[9],  (src[9]  + src[10]) * 20 - (src[8]  + src[11]) * 6 + (src[7]  + src[12]) * 3 - (src[6]  + src[13])); \
888         OP(dst[10], (src[10] + src[11]) * 20 - (src[9]  + src[12]) * 6 + (src[8]  + src[13]) * 3 - (src[7]  + src[14])); \
889         OP(dst[11], (src[11] + src[12]) * 20 - (src[10] + src[13]) * 6 + (src[9]  + src[14]) * 3 - (src[8]  + src[15])); \
890         OP(dst[12], (src[12] + src[13]) * 20 - (src[11] + src[14]) * 6 + (src[10] + src[15]) * 3 - (src[9]  + src[16])); \
891         OP(dst[13], (src[13] + src[14]) * 20 - (src[12] + src[15]) * 6 + (src[11] + src[16]) * 3 - (src[10] + src[16])); \
892         OP(dst[14], (src[14] + src[15]) * 20 - (src[13] + src[16]) * 6 + (src[12] + src[16]) * 3 - (src[11] + src[15])); \
893         OP(dst[15], (src[15] + src[16]) * 20 - (src[14] + src[16]) * 6 + (src[13] + src[15]) * 3 - (src[12] + src[14])); \
894         dst += dstStride;                                                     \
895         src += srcStride;                                                     \
896     }                                                                         \
897 }                                                                             \
898                                                                               \
899 static void OPNAME ## mpeg4_qpel16_v_lowpass(uint8_t *dst, uint8_t *src,      \
900                                              int dstStride, int srcStride)    \
901 {                                                                             \
902     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;                           \
903     const int w = 16;                                                         \
904     int i;                                                                    \
905                                                                               \
906     for (i = 0; i < w; i++) {                                                 \
907         const int src0  = src[0  * srcStride];                                \
908         const int src1  = src[1  * srcStride];                                \
909         const int src2  = src[2  * srcStride];                                \
910         const int src3  = src[3  * srcStride];                                \
911         const int src4  = src[4  * srcStride];                                \
912         const int src5  = src[5  * srcStride];                                \
913         const int src6  = src[6  * srcStride];                                \
914         const int src7  = src[7  * srcStride];                                \
915         const int src8  = src[8  * srcStride];                                \
916         const int src9  = src[9  * srcStride];                                \
917         const int src10 = src[10 * srcStride];                                \
918         const int src11 = src[11 * srcStride];                                \
919         const int src12 = src[12 * srcStride];                                \
920         const int src13 = src[13 * srcStride];                                \
921         const int src14 = src[14 * srcStride];                                \
922         const int src15 = src[15 * srcStride];                                \
923         const int src16 = src[16 * srcStride];                                \
924         OP(dst[0  * dstStride], (src0  + src1)  * 20 - (src0  + src2)  * 6 + (src1  + src3)  * 3 - (src2  + src4));  \
925         OP(dst[1  * dstStride], (src1  + src2)  * 20 - (src0  + src3)  * 6 + (src0  + src4)  * 3 - (src1  + src5));  \
926         OP(dst[2  * dstStride], (src2  + src3)  * 20 - (src1  + src4)  * 6 + (src0  + src5)  * 3 - (src0  + src6));  \
927         OP(dst[3  * dstStride], (src3  + src4)  * 20 - (src2  + src5)  * 6 + (src1  + src6)  * 3 - (src0  + src7));  \
928         OP(dst[4  * dstStride], (src4  + src5)  * 20 - (src3  + src6)  * 6 + (src2  + src7)  * 3 - (src1  + src8));  \
929         OP(dst[5  * dstStride], (src5  + src6)  * 20 - (src4  + src7)  * 6 + (src3  + src8)  * 3 - (src2  + src9));  \
930         OP(dst[6  * dstStride], (src6  + src7)  * 20 - (src5  + src8)  * 6 + (src4  + src9)  * 3 - (src3  + src10)); \
931         OP(dst[7  * dstStride], (src7  + src8)  * 20 - (src6  + src9)  * 6 + (src5  + src10) * 3 - (src4  + src11)); \
932         OP(dst[8  * dstStride], (src8  + src9)  * 20 - (src7  + src10) * 6 + (src6  + src11) * 3 - (src5  + src12)); \
933         OP(dst[9  * dstStride], (src9  + src10) * 20 - (src8  + src11) * 6 + (src7  + src12) * 3 - (src6  + src13)); \
934         OP(dst[10 * dstStride], (src10 + src11) * 20 - (src9  + src12) * 6 + (src8  + src13) * 3 - (src7  + src14)); \
935         OP(dst[11 * dstStride], (src11 + src12) * 20 - (src10 + src13) * 6 + (src9  + src14) * 3 - (src8  + src15)); \
936         OP(dst[12 * dstStride], (src12 + src13) * 20 - (src11 + src14) * 6 + (src10 + src15) * 3 - (src9  + src16)); \
937         OP(dst[13 * dstStride], (src13 + src14) * 20 - (src12 + src15) * 6 + (src11 + src16) * 3 - (src10 + src16)); \
938         OP(dst[14 * dstStride], (src14 + src15) * 20 - (src13 + src16) * 6 + (src12 + src16) * 3 - (src11 + src15)); \
939         OP(dst[15 * dstStride], (src15 + src16) * 20 - (src14 + src16) * 6 + (src13 + src15) * 3 - (src12 + src14)); \
940         dst++;                                                                \
941         src++;                                                                \
942     }                                                                         \
943 }                                                                             \
944                                                                               \
945 static void OPNAME ## qpel8_mc10_c(uint8_t *dst, uint8_t *src,                \
946                                    ptrdiff_t stride)                          \
947 {                                                                             \
948     uint8_t half[64];                                                         \
949                                                                               \
950     put ## RND ## mpeg4_qpel8_h_lowpass(half, src, 8, stride, 8);             \
951     OPNAME ## pixels8_l2_8(dst, src, half, stride, stride, 8, 8);             \
952 }                                                                             \
953                                                                               \
954 static void OPNAME ## qpel8_mc20_c(uint8_t *dst, uint8_t *src,                \
955                                    ptrdiff_t stride)                          \
956 {                                                                             \
957     OPNAME ## mpeg4_qpel8_h_lowpass(dst, src, stride, stride, 8);             \
958 }                                                                             \
959                                                                               \
960 static void OPNAME ## qpel8_mc30_c(uint8_t *dst, uint8_t *src,                \
961                                    ptrdiff_t stride)                          \
962 {                                                                             \
963     uint8_t half[64];                                                         \
964                                                                               \
965     put ## RND ## mpeg4_qpel8_h_lowpass(half, src, 8, stride, 8);             \
966     OPNAME ## pixels8_l2_8(dst, src + 1, half, stride, stride, 8, 8);         \
967 }                                                                             \
968                                                                               \
969 static void OPNAME ## qpel8_mc01_c(uint8_t *dst, uint8_t *src,                \
970                                    ptrdiff_t stride)                          \
971 {                                                                             \
972     uint8_t full[16 * 9];                                                     \
973     uint8_t half[64];                                                         \
974                                                                               \
975     copy_block9(full, src, 16, stride, 9);                                    \
976     put ## RND ## mpeg4_qpel8_v_lowpass(half, full, 8, 16);                   \
977     OPNAME ## pixels8_l2_8(dst, full, half, stride, 16, 8, 8);                \
978 }                                                                             \
979                                                                               \
980 static void OPNAME ## qpel8_mc02_c(uint8_t *dst, uint8_t *src,                \
981                                    ptrdiff_t stride)                          \
982 {                                                                             \
983     uint8_t full[16 * 9];                                                     \
984                                                                               \
985     copy_block9(full, src, 16, stride, 9);                                    \
986     OPNAME ## mpeg4_qpel8_v_lowpass(dst, full, stride, 16);                   \
987 }                                                                             \
988                                                                               \
989 static void OPNAME ## qpel8_mc03_c(uint8_t *dst, uint8_t *src,                \
990                                    ptrdiff_t stride)                          \
991 {                                                                             \
992     uint8_t full[16 * 9];                                                     \
993     uint8_t half[64];                                                         \
994                                                                               \
995     copy_block9(full, src, 16, stride, 9);                                    \
996     put ## RND ## mpeg4_qpel8_v_lowpass(half, full, 8, 16);                   \
997     OPNAME ## pixels8_l2_8(dst, full + 16, half, stride, 16, 8, 8);           \
998 }                                                                             \
999                                                                               \
1000 void ff_ ## OPNAME ## qpel8_mc11_old_c(uint8_t *dst, uint8_t *src,            \
1001                                        ptrdiff_t stride)                      \
1002 {                                                                             \
1003     uint8_t full[16 * 9];                                                     \
1004     uint8_t halfH[72];                                                        \
1005     uint8_t halfV[64];                                                        \
1006     uint8_t halfHV[64];                                                       \
1007                                                                               \
1008     copy_block9(full, src, 16, stride, 9);                                    \
1009     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1010     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full, 8, 16);                  \
1011     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1012     OPNAME ## pixels8_l4_8(dst, full, halfH, halfV, halfHV,                   \
1013                            stride, 16, 8, 8, 8, 8);                           \
1014 }                                                                             \
1015                                                                               \
1016 static void OPNAME ## qpel8_mc11_c(uint8_t *dst, uint8_t *src,                \
1017                                    ptrdiff_t stride)                          \
1018 {                                                                             \
1019     uint8_t full[16 * 9];                                                     \
1020     uint8_t halfH[72];                                                        \
1021     uint8_t halfHV[64];                                                       \
1022                                                                               \
1023     copy_block9(full, src, 16, stride, 9);                                    \
1024     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1025     put ## RND ## pixels8_l2_8(halfH, halfH, full, 8, 8, 16, 9);              \
1026     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1027     OPNAME ## pixels8_l2_8(dst, halfH, halfHV, stride, 8, 8, 8);              \
1028 }                                                                             \
1029                                                                               \
1030 void ff_ ## OPNAME ## qpel8_mc31_old_c(uint8_t *dst, uint8_t *src,            \
1031                                        ptrdiff_t stride)                      \
1032 {                                                                             \
1033     uint8_t full[16 * 9];                                                     \
1034     uint8_t halfH[72];                                                        \
1035     uint8_t halfV[64];                                                        \
1036     uint8_t halfHV[64];                                                       \
1037                                                                               \
1038     copy_block9(full, src, 16, stride, 9);                                    \
1039     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1040     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full + 1, 8, 16);              \
1041     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1042     OPNAME ## pixels8_l4_8(dst, full + 1, halfH, halfV, halfHV,               \
1043                            stride, 16, 8, 8, 8, 8);                           \
1044 }                                                                             \
1045                                                                               \
1046 static void OPNAME ## qpel8_mc31_c(uint8_t *dst, uint8_t *src,                \
1047                                    ptrdiff_t stride)                          \
1048 {                                                                             \
1049     uint8_t full[16 * 9];                                                     \
1050     uint8_t halfH[72];                                                        \
1051     uint8_t halfHV[64];                                                       \
1052                                                                               \
1053     copy_block9(full, src, 16, stride, 9);                                    \
1054     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1055     put ## RND ## pixels8_l2_8(halfH, halfH, full + 1, 8, 8, 16, 9);          \
1056     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1057     OPNAME ## pixels8_l2_8(dst, halfH, halfHV, stride, 8, 8, 8);              \
1058 }                                                                             \
1059                                                                               \
1060 void ff_ ## OPNAME ## qpel8_mc13_old_c(uint8_t *dst, uint8_t *src,            \
1061                                        ptrdiff_t stride)                      \
1062 {                                                                             \
1063     uint8_t full[16 * 9];                                                     \
1064     uint8_t halfH[72];                                                        \
1065     uint8_t halfV[64];                                                        \
1066     uint8_t halfHV[64];                                                       \
1067                                                                               \
1068     copy_block9(full, src, 16, stride, 9);                                    \
1069     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1070     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full, 8, 16);                  \
1071     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1072     OPNAME ## pixels8_l4_8(dst, full + 16, halfH + 8, halfV, halfHV,          \
1073                            stride, 16, 8, 8, 8, 8);                           \
1074 }                                                                             \
1075                                                                               \
1076 static void OPNAME ## qpel8_mc13_c(uint8_t *dst, uint8_t *src,                \
1077                                    ptrdiff_t stride)                          \
1078 {                                                                             \
1079     uint8_t full[16 * 9];                                                     \
1080     uint8_t halfH[72];                                                        \
1081     uint8_t halfHV[64];                                                       \
1082                                                                               \
1083     copy_block9(full, src, 16, stride, 9);                                    \
1084     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1085     put ## RND ## pixels8_l2_8(halfH, halfH, full, 8, 8, 16, 9);              \
1086     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1087     OPNAME ## pixels8_l2_8(dst, halfH + 8, halfHV, stride, 8, 8, 8);          \
1088 }                                                                             \
1089                                                                               \
1090 void ff_ ## OPNAME ## qpel8_mc33_old_c(uint8_t *dst, uint8_t *src,            \
1091                                        ptrdiff_t stride)                      \
1092 {                                                                             \
1093     uint8_t full[16 * 9];                                                     \
1094     uint8_t halfH[72];                                                        \
1095     uint8_t halfV[64];                                                        \
1096     uint8_t halfHV[64];                                                       \
1097                                                                               \
1098     copy_block9(full, src, 16, stride, 9);                                    \
1099     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1100     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full + 1, 8, 16);              \
1101     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1102     OPNAME ## pixels8_l4_8(dst, full + 17, halfH + 8, halfV, halfHV,          \
1103                            stride, 16, 8, 8, 8, 8);                           \
1104 }                                                                             \
1105                                                                               \
1106 static void OPNAME ## qpel8_mc33_c(uint8_t *dst, uint8_t *src,                \
1107                                    ptrdiff_t stride)                          \
1108 {                                                                             \
1109     uint8_t full[16 * 9];                                                     \
1110     uint8_t halfH[72];                                                        \
1111     uint8_t halfHV[64];                                                       \
1112                                                                               \
1113     copy_block9(full, src, 16, stride, 9);                                    \
1114     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1115     put ## RND ## pixels8_l2_8(halfH, halfH, full + 1, 8, 8, 16, 9);          \
1116     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1117     OPNAME ## pixels8_l2_8(dst, halfH + 8, halfHV, stride, 8, 8, 8);          \
1118 }                                                                             \
1119                                                                               \
1120 static void OPNAME ## qpel8_mc21_c(uint8_t *dst, uint8_t *src,                \
1121                                    ptrdiff_t stride)                          \
1122 {                                                                             \
1123     uint8_t halfH[72];                                                        \
1124     uint8_t halfHV[64];                                                       \
1125                                                                               \
1126     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, src, 8, stride, 9);            \
1127     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1128     OPNAME ## pixels8_l2_8(dst, halfH, halfHV, stride, 8, 8, 8);              \
1129 }                                                                             \
1130                                                                               \
1131 static void OPNAME ## qpel8_mc23_c(uint8_t *dst, uint8_t *src,                \
1132                                    ptrdiff_t stride)                          \
1133 {                                                                             \
1134     uint8_t halfH[72];                                                        \
1135     uint8_t halfHV[64];                                                       \
1136                                                                               \
1137     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, src, 8, stride, 9);            \
1138     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1139     OPNAME ## pixels8_l2_8(dst, halfH + 8, halfHV, stride, 8, 8, 8);          \
1140 }                                                                             \
1141                                                                               \
1142 void ff_ ## OPNAME ## qpel8_mc12_old_c(uint8_t *dst, uint8_t *src,            \
1143                                        ptrdiff_t stride)                      \
1144 {                                                                             \
1145     uint8_t full[16 * 9];                                                     \
1146     uint8_t halfH[72];                                                        \
1147     uint8_t halfV[64];                                                        \
1148     uint8_t halfHV[64];                                                       \
1149                                                                               \
1150     copy_block9(full, src, 16, stride, 9);                                    \
1151     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1152     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full, 8, 16);                  \
1153     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1154     OPNAME ## pixels8_l2_8(dst, halfV, halfHV, stride, 8, 8, 8);              \
1155 }                                                                             \
1156                                                                               \
1157 static void OPNAME ## qpel8_mc12_c(uint8_t *dst, uint8_t *src,                \
1158                                    ptrdiff_t stride)                          \
1159 {                                                                             \
1160     uint8_t full[16 * 9];                                                     \
1161     uint8_t halfH[72];                                                        \
1162                                                                               \
1163     copy_block9(full, src, 16, stride, 9);                                    \
1164     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1165     put ## RND ## pixels8_l2_8(halfH, halfH, full, 8, 8, 16, 9);              \
1166     OPNAME ## mpeg4_qpel8_v_lowpass(dst, halfH, stride, 8);                   \
1167 }                                                                             \
1168                                                                               \
1169 void ff_ ## OPNAME ## qpel8_mc32_old_c(uint8_t *dst, uint8_t *src,            \
1170                                        ptrdiff_t stride)                      \
1171 {                                                                             \
1172     uint8_t full[16 * 9];                                                     \
1173     uint8_t halfH[72];                                                        \
1174     uint8_t halfV[64];                                                        \
1175     uint8_t halfHV[64];                                                       \
1176                                                                               \
1177     copy_block9(full, src, 16, stride, 9);                                    \
1178     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1179     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full + 1, 8, 16);              \
1180     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
1181     OPNAME ## pixels8_l2_8(dst, halfV, halfHV, stride, 8, 8, 8);              \
1182 }                                                                             \
1183                                                                               \
1184 static void OPNAME ## qpel8_mc32_c(uint8_t *dst, uint8_t *src,                \
1185                                    ptrdiff_t stride)                          \
1186 {                                                                             \
1187     uint8_t full[16 * 9];                                                     \
1188     uint8_t halfH[72];                                                        \
1189                                                                               \
1190     copy_block9(full, src, 16, stride, 9);                                    \
1191     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
1192     put ## RND ## pixels8_l2_8(halfH, halfH, full + 1, 8, 8, 16, 9);          \
1193     OPNAME ## mpeg4_qpel8_v_lowpass(dst, halfH, stride, 8);                   \
1194 }                                                                             \
1195                                                                               \
1196 static void OPNAME ## qpel8_mc22_c(uint8_t *dst, uint8_t *src,                \
1197                                    ptrdiff_t stride)                          \
1198 {                                                                             \
1199     uint8_t halfH[72];                                                        \
1200                                                                               \
1201     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, src, 8, stride, 9);            \
1202     OPNAME ## mpeg4_qpel8_v_lowpass(dst, halfH, stride, 8);                   \
1203 }                                                                             \
1204                                                                               \
1205 static void OPNAME ## qpel16_mc10_c(uint8_t *dst, uint8_t *src,               \
1206                                     ptrdiff_t stride)                         \
1207 {                                                                             \
1208     uint8_t half[256];                                                        \
1209                                                                               \
1210     put ## RND ## mpeg4_qpel16_h_lowpass(half, src, 16, stride, 16);          \
1211     OPNAME ## pixels16_l2_8(dst, src, half, stride, stride, 16, 16);          \
1212 }                                                                             \
1213                                                                               \
1214 static void OPNAME ## qpel16_mc20_c(uint8_t *dst, uint8_t *src,               \
1215                                     ptrdiff_t stride)                         \
1216 {                                                                             \
1217     OPNAME ## mpeg4_qpel16_h_lowpass(dst, src, stride, stride, 16);           \
1218 }                                                                             \
1219                                                                               \
1220 static void OPNAME ## qpel16_mc30_c(uint8_t *dst, uint8_t *src,               \
1221                                     ptrdiff_t stride)                         \
1222 {                                                                             \
1223     uint8_t half[256];                                                        \
1224                                                                               \
1225     put ## RND ## mpeg4_qpel16_h_lowpass(half, src, 16, stride, 16);          \
1226     OPNAME ## pixels16_l2_8(dst, src + 1, half, stride, stride, 16, 16);      \
1227 }                                                                             \
1228                                                                               \
1229 static void OPNAME ## qpel16_mc01_c(uint8_t *dst, uint8_t *src,               \
1230                                     ptrdiff_t stride)                         \
1231 {                                                                             \
1232     uint8_t full[24 * 17];                                                    \
1233     uint8_t half[256];                                                        \
1234                                                                               \
1235     copy_block17(full, src, 24, stride, 17);                                  \
1236     put ## RND ## mpeg4_qpel16_v_lowpass(half, full, 16, 24);                 \
1237     OPNAME ## pixels16_l2_8(dst, full, half, stride, 24, 16, 16);             \
1238 }                                                                             \
1239                                                                               \
1240 static void OPNAME ## qpel16_mc02_c(uint8_t *dst, uint8_t *src,               \
1241                                     ptrdiff_t stride)                         \
1242 {                                                                             \
1243     uint8_t full[24 * 17];                                                    \
1244                                                                               \
1245     copy_block17(full, src, 24, stride, 17);                                  \
1246     OPNAME ## mpeg4_qpel16_v_lowpass(dst, full, stride, 24);                  \
1247 }                                                                             \
1248                                                                               \
1249 static void OPNAME ## qpel16_mc03_c(uint8_t *dst, uint8_t *src,               \
1250                                     ptrdiff_t stride)                         \
1251 {                                                                             \
1252     uint8_t full[24 * 17];                                                    \
1253     uint8_t half[256];                                                        \
1254                                                                               \
1255     copy_block17(full, src, 24, stride, 17);                                  \
1256     put ## RND ## mpeg4_qpel16_v_lowpass(half, full, 16, 24);                 \
1257     OPNAME ## pixels16_l2_8(dst, full + 24, half, stride, 24, 16, 16);        \
1258 }                                                                             \
1259                                                                               \
1260 void ff_ ## OPNAME ## qpel16_mc11_old_c(uint8_t *dst, uint8_t *src,           \
1261                                         ptrdiff_t stride)                     \
1262 {                                                                             \
1263     uint8_t full[24 * 17];                                                    \
1264     uint8_t halfH[272];                                                       \
1265     uint8_t halfV[256];                                                       \
1266     uint8_t halfHV[256];                                                      \
1267                                                                               \
1268     copy_block17(full, src, 24, stride, 17);                                  \
1269     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1270     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full, 16, 24);                \
1271     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1272     OPNAME ## pixels16_l4_8(dst, full, halfH, halfV, halfHV,                  \
1273                             stride, 24, 16, 16, 16, 16);                      \
1274 }                                                                             \
1275                                                                               \
1276 static void OPNAME ## qpel16_mc11_c(uint8_t *dst, uint8_t *src,               \
1277                                     ptrdiff_t stride)                         \
1278 {                                                                             \
1279     uint8_t full[24 * 17];                                                    \
1280     uint8_t halfH[272];                                                       \
1281     uint8_t halfHV[256];                                                      \
1282                                                                               \
1283     copy_block17(full, src, 24, stride, 17);                                  \
1284     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1285     put ## RND ## pixels16_l2_8(halfH, halfH, full, 16, 16, 24, 17);          \
1286     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1287     OPNAME ## pixels16_l2_8(dst, halfH, halfHV, stride, 16, 16, 16);          \
1288 }                                                                             \
1289                                                                               \
1290 void ff_ ## OPNAME ## qpel16_mc31_old_c(uint8_t *dst, uint8_t *src,           \
1291                                         ptrdiff_t stride)                     \
1292 {                                                                             \
1293     uint8_t full[24 * 17];                                                    \
1294     uint8_t halfH[272];                                                       \
1295     uint8_t halfV[256];                                                       \
1296     uint8_t halfHV[256];                                                      \
1297                                                                               \
1298     copy_block17(full, src, 24, stride, 17);                                  \
1299     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1300     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full + 1, 16, 24);            \
1301     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1302     OPNAME ## pixels16_l4_8(dst, full + 1, halfH, halfV, halfHV,              \
1303                             stride, 24, 16, 16, 16, 16);                      \
1304 }                                                                             \
1305                                                                               \
1306 static void OPNAME ## qpel16_mc31_c(uint8_t *dst, uint8_t *src,               \
1307                                     ptrdiff_t stride)                         \
1308 {                                                                             \
1309     uint8_t full[24 * 17];                                                    \
1310     uint8_t halfH[272];                                                       \
1311     uint8_t halfHV[256];                                                      \
1312                                                                               \
1313     copy_block17(full, src, 24, stride, 17);                                  \
1314     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1315     put ## RND ## pixels16_l2_8(halfH, halfH, full + 1, 16, 16, 24, 17);      \
1316     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1317     OPNAME ## pixels16_l2_8(dst, halfH, halfHV, stride, 16, 16, 16);          \
1318 }                                                                             \
1319                                                                               \
1320 void ff_ ## OPNAME ## qpel16_mc13_old_c(uint8_t *dst, uint8_t *src,           \
1321                                         ptrdiff_t stride)                     \
1322 {                                                                             \
1323     uint8_t full[24 * 17];                                                    \
1324     uint8_t halfH[272];                                                       \
1325     uint8_t halfV[256];                                                       \
1326     uint8_t halfHV[256];                                                      \
1327                                                                               \
1328     copy_block17(full, src, 24, stride, 17);                                  \
1329     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1330     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full, 16, 24);                \
1331     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1332     OPNAME ## pixels16_l4_8(dst, full + 24, halfH + 16, halfV, halfHV,        \
1333                             stride, 24, 16, 16, 16, 16);                      \
1334 }                                                                             \
1335                                                                               \
1336 static void OPNAME ## qpel16_mc13_c(uint8_t *dst, uint8_t *src,               \
1337                                     ptrdiff_t stride)                         \
1338 {                                                                             \
1339     uint8_t full[24 * 17];                                                    \
1340     uint8_t halfH[272];                                                       \
1341     uint8_t halfHV[256];                                                      \
1342                                                                               \
1343     copy_block17(full, src, 24, stride, 17);                                  \
1344     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1345     put ## RND ## pixels16_l2_8(halfH, halfH, full, 16, 16, 24, 17);          \
1346     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1347     OPNAME ## pixels16_l2_8(dst, halfH + 16, halfHV, stride, 16, 16, 16);     \
1348 }                                                                             \
1349                                                                               \
1350 void ff_ ## OPNAME ## qpel16_mc33_old_c(uint8_t *dst, uint8_t *src,           \
1351                                         ptrdiff_t stride)                     \
1352 {                                                                             \
1353     uint8_t full[24 * 17];                                                    \
1354     uint8_t halfH[272];                                                       \
1355     uint8_t halfV[256];                                                       \
1356     uint8_t halfHV[256];                                                      \
1357                                                                               \
1358     copy_block17(full, src, 24, stride, 17);                                  \
1359     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1360     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full + 1, 16, 24);            \
1361     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1362     OPNAME ## pixels16_l4_8(dst, full + 25, halfH + 16, halfV, halfHV,        \
1363                             stride, 24, 16, 16, 16, 16);                      \
1364 }                                                                             \
1365                                                                               \
1366 static void OPNAME ## qpel16_mc33_c(uint8_t *dst, uint8_t *src,               \
1367                                     ptrdiff_t stride)                         \
1368 {                                                                             \
1369     uint8_t full[24 * 17];                                                    \
1370     uint8_t halfH[272];                                                       \
1371     uint8_t halfHV[256];                                                      \
1372                                                                               \
1373     copy_block17(full, src, 24, stride, 17);                                  \
1374     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1375     put ## RND ## pixels16_l2_8(halfH, halfH, full + 1, 16, 16, 24, 17);      \
1376     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1377     OPNAME ## pixels16_l2_8(dst, halfH + 16, halfHV, stride, 16, 16, 16);     \
1378 }                                                                             \
1379                                                                               \
1380 static void OPNAME ## qpel16_mc21_c(uint8_t *dst, uint8_t *src,               \
1381                                     ptrdiff_t stride)                         \
1382 {                                                                             \
1383     uint8_t halfH[272];                                                       \
1384     uint8_t halfHV[256];                                                      \
1385                                                                               \
1386     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, src, 16, stride, 17);         \
1387     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1388     OPNAME ## pixels16_l2_8(dst, halfH, halfHV, stride, 16, 16, 16);          \
1389 }                                                                             \
1390                                                                               \
1391 static void OPNAME ## qpel16_mc23_c(uint8_t *dst, uint8_t *src,               \
1392                                     ptrdiff_t stride)                         \
1393 {                                                                             \
1394     uint8_t halfH[272];                                                       \
1395     uint8_t halfHV[256];                                                      \
1396                                                                               \
1397     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, src, 16, stride, 17);         \
1398     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1399     OPNAME ## pixels16_l2_8(dst, halfH + 16, halfHV, stride, 16, 16, 16);     \
1400 }                                                                             \
1401                                                                               \
1402 void ff_ ## OPNAME ## qpel16_mc12_old_c(uint8_t *dst, uint8_t *src,           \
1403                                         ptrdiff_t stride)                     \
1404 {                                                                             \
1405     uint8_t full[24 * 17];                                                    \
1406     uint8_t halfH[272];                                                       \
1407     uint8_t halfV[256];                                                       \
1408     uint8_t halfHV[256];                                                      \
1409                                                                               \
1410     copy_block17(full, src, 24, stride, 17);                                  \
1411     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1412     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full, 16, 24);                \
1413     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1414     OPNAME ## pixels16_l2_8(dst, halfV, halfHV, stride, 16, 16, 16);          \
1415 }                                                                             \
1416                                                                               \
1417 static void OPNAME ## qpel16_mc12_c(uint8_t *dst, uint8_t *src,               \
1418                                     ptrdiff_t stride)                         \
1419 {                                                                             \
1420     uint8_t full[24 * 17];                                                    \
1421     uint8_t halfH[272];                                                       \
1422                                                                               \
1423     copy_block17(full, src, 24, stride, 17);                                  \
1424     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1425     put ## RND ## pixels16_l2_8(halfH, halfH, full, 16, 16, 24, 17);          \
1426     OPNAME ## mpeg4_qpel16_v_lowpass(dst, halfH, stride, 16);                 \
1427 }                                                                             \
1428                                                                               \
1429 void ff_ ## OPNAME ## qpel16_mc32_old_c(uint8_t *dst, uint8_t *src,           \
1430                                         ptrdiff_t stride)                     \
1431 {                                                                             \
1432     uint8_t full[24 * 17];                                                    \
1433     uint8_t halfH[272];                                                       \
1434     uint8_t halfV[256];                                                       \
1435     uint8_t halfHV[256];                                                      \
1436                                                                               \
1437     copy_block17(full, src, 24, stride, 17);                                  \
1438     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1439     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full + 1, 16, 24);            \
1440     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1441     OPNAME ## pixels16_l2_8(dst, halfV, halfHV, stride, 16, 16, 16);          \
1442 }                                                                             \
1443                                                                               \
1444 static void OPNAME ## qpel16_mc32_c(uint8_t *dst, uint8_t *src,               \
1445                                     ptrdiff_t stride)                         \
1446 {                                                                             \
1447     uint8_t full[24 * 17];                                                    \
1448     uint8_t halfH[272];                                                       \
1449                                                                               \
1450     copy_block17(full, src, 24, stride, 17);                                  \
1451     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1452     put ## RND ## pixels16_l2_8(halfH, halfH, full + 1, 16, 16, 24, 17);      \
1453     OPNAME ## mpeg4_qpel16_v_lowpass(dst, halfH, stride, 16);                 \
1454 }                                                                             \
1455                                                                               \
1456 static void OPNAME ## qpel16_mc22_c(uint8_t *dst, uint8_t *src,               \
1457                                     ptrdiff_t stride)                         \
1458 {                                                                             \
1459     uint8_t halfH[272];                                                       \
1460                                                                               \
1461     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, src, 16, stride, 17);         \
1462     OPNAME ## mpeg4_qpel16_v_lowpass(dst, halfH, stride, 16);                 \
1463 }
1464
1465 #define op_avg(a, b)        a = (((a) + cm[((b) + 16) >> 5] + 1) >> 1)
1466 #define op_avg_no_rnd(a, b) a = (((a) + cm[((b) + 15) >> 5])     >> 1)
1467 #define op_put(a, b)        a = cm[((b) + 16) >> 5]
1468 #define op_put_no_rnd(a, b) a = cm[((b) + 15) >> 5]
1469
1470 QPEL_MC(0, put_, _, op_put)
1471 QPEL_MC(1, put_no_rnd_, _no_rnd_, op_put_no_rnd)
1472 QPEL_MC(0, avg_, _, op_avg)
1473
1474 #undef op_avg
1475 #undef op_put
1476 #undef op_put_no_rnd
1477
1478 void ff_put_pixels8x8_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1479 {
1480     put_pixels8_8_c(dst, src, stride, 8);
1481 }
1482
1483 void ff_avg_pixels8x8_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1484 {
1485     avg_pixels8_8_c(dst, src, stride, 8);
1486 }
1487
1488 void ff_put_pixels16x16_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1489 {
1490     put_pixels16_8_c(dst, src, stride, 16);
1491 }
1492
1493 void ff_avg_pixels16x16_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1494 {
1495     avg_pixels16_8_c(dst, src, stride, 16);
1496 }
1497
1498 #define put_qpel8_mc00_c         ff_put_pixels8x8_c
1499 #define avg_qpel8_mc00_c         ff_avg_pixels8x8_c
1500 #define put_qpel16_mc00_c        ff_put_pixels16x16_c
1501 #define avg_qpel16_mc00_c        ff_avg_pixels16x16_c
1502 #define put_no_rnd_qpel8_mc00_c  ff_put_pixels8x8_c
1503 #define put_no_rnd_qpel16_mc00_c ff_put_pixels16x16_c
1504
1505 static void wmv2_mspel8_h_lowpass(uint8_t *dst, uint8_t *src,
1506                                   int dstStride, int srcStride, int h)
1507 {
1508     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;
1509     int i;
1510
1511     for (i = 0; i < h; i++) {
1512         dst[0] = cm[(9 * (src[0] + src[1]) - (src[-1] + src[2]) + 8) >> 4];
1513         dst[1] = cm[(9 * (src[1] + src[2]) - (src[0]  + src[3]) + 8) >> 4];
1514         dst[2] = cm[(9 * (src[2] + src[3]) - (src[1]  + src[4]) + 8) >> 4];
1515         dst[3] = cm[(9 * (src[3] + src[4]) - (src[2]  + src[5]) + 8) >> 4];
1516         dst[4] = cm[(9 * (src[4] + src[5]) - (src[3]  + src[6]) + 8) >> 4];
1517         dst[5] = cm[(9 * (src[5] + src[6]) - (src[4]  + src[7]) + 8) >> 4];
1518         dst[6] = cm[(9 * (src[6] + src[7]) - (src[5]  + src[8]) + 8) >> 4];
1519         dst[7] = cm[(9 * (src[7] + src[8]) - (src[6]  + src[9]) + 8) >> 4];
1520         dst   += dstStride;
1521         src   += srcStride;
1522     }
1523 }
1524
1525 #if CONFIG_RV40_DECODER
1526 void ff_put_rv40_qpel16_mc33_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1527 {
1528     put_pixels16_xy2_8_c(dst, src, stride, 16);
1529 }
1530
1531 void ff_avg_rv40_qpel16_mc33_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1532 {
1533     avg_pixels16_xy2_8_c(dst, src, stride, 16);
1534 }
1535
1536 void ff_put_rv40_qpel8_mc33_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1537 {
1538     put_pixels8_xy2_8_c(dst, src, stride, 8);
1539 }
1540
1541 void ff_avg_rv40_qpel8_mc33_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1542 {
1543     avg_pixels8_xy2_8_c(dst, src, stride, 8);
1544 }
1545 #endif /* CONFIG_RV40_DECODER */
1546
1547 static void wmv2_mspel8_v_lowpass(uint8_t *dst, uint8_t *src,
1548                                   int dstStride, int srcStride, int w)
1549 {
1550     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;
1551     int i;
1552
1553     for (i = 0; i < w; i++) {
1554         const int src_1 = src[-srcStride];
1555         const int src0  = src[0];
1556         const int src1  = src[srcStride];
1557         const int src2  = src[2 * srcStride];
1558         const int src3  = src[3 * srcStride];
1559         const int src4  = src[4 * srcStride];
1560         const int src5  = src[5 * srcStride];
1561         const int src6  = src[6 * srcStride];
1562         const int src7  = src[7 * srcStride];
1563         const int src8  = src[8 * srcStride];
1564         const int src9  = src[9 * srcStride];
1565         dst[0 * dstStride] = cm[(9 * (src0 + src1) - (src_1 + src2) + 8) >> 4];
1566         dst[1 * dstStride] = cm[(9 * (src1 + src2) - (src0  + src3) + 8) >> 4];
1567         dst[2 * dstStride] = cm[(9 * (src2 + src3) - (src1  + src4) + 8) >> 4];
1568         dst[3 * dstStride] = cm[(9 * (src3 + src4) - (src2  + src5) + 8) >> 4];
1569         dst[4 * dstStride] = cm[(9 * (src4 + src5) - (src3  + src6) + 8) >> 4];
1570         dst[5 * dstStride] = cm[(9 * (src5 + src6) - (src4  + src7) + 8) >> 4];
1571         dst[6 * dstStride] = cm[(9 * (src6 + src7) - (src5  + src8) + 8) >> 4];
1572         dst[7 * dstStride] = cm[(9 * (src7 + src8) - (src6  + src9) + 8) >> 4];
1573         src++;
1574         dst++;
1575     }
1576 }
1577
1578 static void put_mspel8_mc10_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1579 {
1580     uint8_t half[64];
1581
1582     wmv2_mspel8_h_lowpass(half, src, 8, stride, 8);
1583     put_pixels8_l2_8(dst, src, half, stride, stride, 8, 8);
1584 }
1585
1586 static void put_mspel8_mc20_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1587 {
1588     wmv2_mspel8_h_lowpass(dst, src, stride, stride, 8);
1589 }
1590
1591 static void put_mspel8_mc30_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1592 {
1593     uint8_t half[64];
1594
1595     wmv2_mspel8_h_lowpass(half, src, 8, stride, 8);
1596     put_pixels8_l2_8(dst, src + 1, half, stride, stride, 8, 8);
1597 }
1598
1599 static void put_mspel8_mc02_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1600 {
1601     wmv2_mspel8_v_lowpass(dst, src, stride, stride, 8);
1602 }
1603
1604 static void put_mspel8_mc12_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1605 {
1606     uint8_t halfH[88];
1607     uint8_t halfV[64];
1608     uint8_t halfHV[64];
1609
1610     wmv2_mspel8_h_lowpass(halfH, src - stride, 8, stride, 11);
1611     wmv2_mspel8_v_lowpass(halfV, src, 8, stride, 8);
1612     wmv2_mspel8_v_lowpass(halfHV, halfH + 8, 8, 8, 8);
1613     put_pixels8_l2_8(dst, halfV, halfHV, stride, 8, 8, 8);
1614 }
1615
1616 static void put_mspel8_mc32_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1617 {
1618     uint8_t halfH[88];
1619     uint8_t halfV[64];
1620     uint8_t halfHV[64];
1621
1622     wmv2_mspel8_h_lowpass(halfH, src - stride, 8, stride, 11);
1623     wmv2_mspel8_v_lowpass(halfV, src + 1, 8, stride, 8);
1624     wmv2_mspel8_v_lowpass(halfHV, halfH + 8, 8, 8, 8);
1625     put_pixels8_l2_8(dst, halfV, halfHV, stride, 8, 8, 8);
1626 }
1627
1628 static void put_mspel8_mc22_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1629 {
1630     uint8_t halfH[88];
1631
1632     wmv2_mspel8_h_lowpass(halfH, src - stride, 8, stride, 11);
1633     wmv2_mspel8_v_lowpass(dst, halfH + 8, stride, 8, 8);
1634 }
1635
1636 static inline int pix_abs16_c(void *v, uint8_t *pix1, uint8_t *pix2,
1637                               int line_size, int h)
1638 {
1639     int s = 0, i;
1640
1641     for (i = 0; i < h; i++) {
1642         s    += abs(pix1[0]  - pix2[0]);
1643         s    += abs(pix1[1]  - pix2[1]);
1644         s    += abs(pix1[2]  - pix2[2]);
1645         s    += abs(pix1[3]  - pix2[3]);
1646         s    += abs(pix1[4]  - pix2[4]);
1647         s    += abs(pix1[5]  - pix2[5]);
1648         s    += abs(pix1[6]  - pix2[6]);
1649         s    += abs(pix1[7]  - pix2[7]);
1650         s    += abs(pix1[8]  - pix2[8]);
1651         s    += abs(pix1[9]  - pix2[9]);
1652         s    += abs(pix1[10] - pix2[10]);
1653         s    += abs(pix1[11] - pix2[11]);
1654         s    += abs(pix1[12] - pix2[12]);
1655         s    += abs(pix1[13] - pix2[13]);
1656         s    += abs(pix1[14] - pix2[14]);
1657         s    += abs(pix1[15] - pix2[15]);
1658         pix1 += line_size;
1659         pix2 += line_size;
1660     }
1661     return s;
1662 }
1663
1664 static int pix_abs16_x2_c(void *v, uint8_t *pix1, uint8_t *pix2,
1665                           int line_size, int h)
1666 {
1667     int s = 0, i;
1668
1669     for (i = 0; i < h; i++) {
1670         s    += abs(pix1[0]  - avg2(pix2[0],  pix2[1]));
1671         s    += abs(pix1[1]  - avg2(pix2[1],  pix2[2]));
1672         s    += abs(pix1[2]  - avg2(pix2[2],  pix2[3]));
1673         s    += abs(pix1[3]  - avg2(pix2[3],  pix2[4]));
1674         s    += abs(pix1[4]  - avg2(pix2[4],  pix2[5]));
1675         s    += abs(pix1[5]  - avg2(pix2[5],  pix2[6]));
1676         s    += abs(pix1[6]  - avg2(pix2[6],  pix2[7]));
1677         s    += abs(pix1[7]  - avg2(pix2[7],  pix2[8]));
1678         s    += abs(pix1[8]  - avg2(pix2[8],  pix2[9]));
1679         s    += abs(pix1[9]  - avg2(pix2[9],  pix2[10]));
1680         s    += abs(pix1[10] - avg2(pix2[10], pix2[11]));
1681         s    += abs(pix1[11] - avg2(pix2[11], pix2[12]));
1682         s    += abs(pix1[12] - avg2(pix2[12], pix2[13]));
1683         s    += abs(pix1[13] - avg2(pix2[13], pix2[14]));
1684         s    += abs(pix1[14] - avg2(pix2[14], pix2[15]));
1685         s    += abs(pix1[15] - avg2(pix2[15], pix2[16]));
1686         pix1 += line_size;
1687         pix2 += line_size;
1688     }
1689     return s;
1690 }
1691
1692 static int pix_abs16_y2_c(void *v, uint8_t *pix1, uint8_t *pix2,
1693                           int line_size, int h)
1694 {
1695     int s = 0, i;
1696     uint8_t *pix3 = pix2 + line_size;
1697
1698     for (i = 0; i < h; i++) {
1699         s    += abs(pix1[0]  - avg2(pix2[0],  pix3[0]));
1700         s    += abs(pix1[1]  - avg2(pix2[1],  pix3[1]));
1701         s    += abs(pix1[2]  - avg2(pix2[2],  pix3[2]));
1702         s    += abs(pix1[3]  - avg2(pix2[3],  pix3[3]));
1703         s    += abs(pix1[4]  - avg2(pix2[4],  pix3[4]));
1704         s    += abs(pix1[5]  - avg2(pix2[5],  pix3[5]));
1705         s    += abs(pix1[6]  - avg2(pix2[6],  pix3[6]));
1706         s    += abs(pix1[7]  - avg2(pix2[7],  pix3[7]));
1707         s    += abs(pix1[8]  - avg2(pix2[8],  pix3[8]));
1708         s    += abs(pix1[9]  - avg2(pix2[9],  pix3[9]));
1709         s    += abs(pix1[10] - avg2(pix2[10], pix3[10]));
1710         s    += abs(pix1[11] - avg2(pix2[11], pix3[11]));
1711         s    += abs(pix1[12] - avg2(pix2[12], pix3[12]));
1712         s    += abs(pix1[13] - avg2(pix2[13], pix3[13]));
1713         s    += abs(pix1[14] - avg2(pix2[14], pix3[14]));
1714         s    += abs(pix1[15] - avg2(pix2[15], pix3[15]));
1715         pix1 += line_size;
1716         pix2 += line_size;
1717         pix3 += line_size;
1718     }
1719     return s;
1720 }
1721
1722 static int pix_abs16_xy2_c(void *v, uint8_t *pix1, uint8_t *pix2,
1723                            int line_size, int h)
1724 {
1725     int s = 0, i;
1726     uint8_t *pix3 = pix2 + line_size;
1727
1728     for (i = 0; i < h; i++) {
1729         s    += abs(pix1[0]  - avg4(pix2[0],  pix2[1],  pix3[0],  pix3[1]));
1730         s    += abs(pix1[1]  - avg4(pix2[1],  pix2[2],  pix3[1],  pix3[2]));
1731         s    += abs(pix1[2]  - avg4(pix2[2],  pix2[3],  pix3[2],  pix3[3]));
1732         s    += abs(pix1[3]  - avg4(pix2[3],  pix2[4],  pix3[3],  pix3[4]));
1733         s    += abs(pix1[4]  - avg4(pix2[4],  pix2[5],  pix3[4],  pix3[5]));
1734         s    += abs(pix1[5]  - avg4(pix2[5],  pix2[6],  pix3[5],  pix3[6]));
1735         s    += abs(pix1[6]  - avg4(pix2[6],  pix2[7],  pix3[6],  pix3[7]));
1736         s    += abs(pix1[7]  - avg4(pix2[7],  pix2[8],  pix3[7],  pix3[8]));
1737         s    += abs(pix1[8]  - avg4(pix2[8],  pix2[9],  pix3[8],  pix3[9]));
1738         s    += abs(pix1[9]  - avg4(pix2[9],  pix2[10], pix3[9],  pix3[10]));
1739         s    += abs(pix1[10] - avg4(pix2[10], pix2[11], pix3[10], pix3[11]));
1740         s    += abs(pix1[11] - avg4(pix2[11], pix2[12], pix3[11], pix3[12]));
1741         s    += abs(pix1[12] - avg4(pix2[12], pix2[13], pix3[12], pix3[13]));
1742         s    += abs(pix1[13] - avg4(pix2[13], pix2[14], pix3[13], pix3[14]));
1743         s    += abs(pix1[14] - avg4(pix2[14], pix2[15], pix3[14], pix3[15]));
1744         s    += abs(pix1[15] - avg4(pix2[15], pix2[16], pix3[15], pix3[16]));
1745         pix1 += line_size;
1746         pix2 += line_size;
1747         pix3 += line_size;
1748     }
1749     return s;
1750 }
1751
1752 static inline int pix_abs8_c(void *v, uint8_t *pix1, uint8_t *pix2,
1753                              int line_size, int h)
1754 {
1755     int s = 0, i;
1756
1757     for (i = 0; i < h; i++) {
1758         s    += abs(pix1[0] - pix2[0]);
1759         s    += abs(pix1[1] - pix2[1]);
1760         s    += abs(pix1[2] - pix2[2]);
1761         s    += abs(pix1[3] - pix2[3]);
1762         s    += abs(pix1[4] - pix2[4]);
1763         s    += abs(pix1[5] - pix2[5]);
1764         s    += abs(pix1[6] - pix2[6]);
1765         s    += abs(pix1[7] - pix2[7]);
1766         pix1 += line_size;
1767         pix2 += line_size;
1768     }
1769     return s;
1770 }
1771
1772 static int pix_abs8_x2_c(void *v, uint8_t *pix1, uint8_t *pix2,
1773                          int line_size, int h)
1774 {
1775     int s = 0, i;
1776
1777     for (i = 0; i < h; i++) {
1778         s    += abs(pix1[0] - avg2(pix2[0], pix2[1]));
1779         s    += abs(pix1[1] - avg2(pix2[1], pix2[2]));
1780         s    += abs(pix1[2] - avg2(pix2[2], pix2[3]));
1781         s    += abs(pix1[3] - avg2(pix2[3], pix2[4]));
1782         s    += abs(pix1[4] - avg2(pix2[4], pix2[5]));
1783         s    += abs(pix1[5] - avg2(pix2[5], pix2[6]));
1784         s    += abs(pix1[6] - avg2(pix2[6], pix2[7]));
1785         s    += abs(pix1[7] - avg2(pix2[7], pix2[8]));
1786         pix1 += line_size;
1787         pix2 += line_size;
1788     }
1789     return s;
1790 }
1791
1792 static int pix_abs8_y2_c(void *v, uint8_t *pix1, uint8_t *pix2,
1793                          int line_size, int h)
1794 {
1795     int s = 0, i;
1796     uint8_t *pix3 = pix2 + line_size;
1797
1798     for (i = 0; i < h; i++) {
1799         s    += abs(pix1[0] - avg2(pix2[0], pix3[0]));
1800         s    += abs(pix1[1] - avg2(pix2[1], pix3[1]));
1801         s    += abs(pix1[2] - avg2(pix2[2], pix3[2]));
1802         s    += abs(pix1[3] - avg2(pix2[3], pix3[3]));
1803         s    += abs(pix1[4] - avg2(pix2[4], pix3[4]));
1804         s    += abs(pix1[5] - avg2(pix2[5], pix3[5]));
1805         s    += abs(pix1[6] - avg2(pix2[6], pix3[6]));
1806         s    += abs(pix1[7] - avg2(pix2[7], pix3[7]));
1807         pix1 += line_size;
1808         pix2 += line_size;
1809         pix3 += line_size;
1810     }
1811     return s;
1812 }
1813
1814 static int pix_abs8_xy2_c(void *v, uint8_t *pix1, uint8_t *pix2,
1815                           int line_size, int h)
1816 {
1817     int s = 0, i;
1818     uint8_t *pix3 = pix2 + line_size;
1819
1820     for (i = 0; i < h; i++) {
1821         s    += abs(pix1[0] - avg4(pix2[0], pix2[1], pix3[0], pix3[1]));
1822         s    += abs(pix1[1] - avg4(pix2[1], pix2[2], pix3[1], pix3[2]));
1823         s    += abs(pix1[2] - avg4(pix2[2], pix2[3], pix3[2], pix3[3]));
1824         s    += abs(pix1[3] - avg4(pix2[3], pix2[4], pix3[3], pix3[4]));
1825         s    += abs(pix1[4] - avg4(pix2[4], pix2[5], pix3[4], pix3[5]));
1826         s    += abs(pix1[5] - avg4(pix2[5], pix2[6], pix3[5], pix3[6]));
1827         s    += abs(pix1[6] - avg4(pix2[6], pix2[7], pix3[6], pix3[7]));
1828         s    += abs(pix1[7] - avg4(pix2[7], pix2[8], pix3[7], pix3[8]));
1829         pix1 += line_size;
1830         pix2 += line_size;
1831         pix3 += line_size;
1832     }
1833     return s;
1834 }
1835
1836 static int nsse16_c(void *v, uint8_t *s1, uint8_t *s2, int stride, int h)
1837 {
1838     MpegEncContext *c = v;
1839     int score1 = 0, score2 = 0, x, y;
1840
1841     for (y = 0; y < h; y++) {
1842         for (x = 0; x < 16; x++)
1843             score1 += (s1[x] - s2[x]) * (s1[x] - s2[x]);
1844         if (y + 1 < h) {
1845             for (x = 0; x < 15; x++)
1846                 score2 += FFABS(s1[x]     - s1[x + stride] -
1847                                 s1[x + 1] + s1[x + stride + 1]) -
1848                           FFABS(s2[x]     - s2[x + stride] -
1849                                 s2[x + 1] + s2[x + stride + 1]);
1850         }
1851         s1 += stride;
1852         s2 += stride;
1853     }
1854
1855     if (c)
1856         return score1 + FFABS(score2) * c->avctx->nsse_weight;
1857     else
1858         return score1 + FFABS(score2) * 8;
1859 }
1860
1861 static int nsse8_c(void *v, uint8_t *s1, uint8_t *s2, int stride, int h)
1862 {
1863     MpegEncContext *c = v;
1864     int score1 = 0, score2 = 0, x, y;
1865
1866     for (y = 0; y < h; y++) {
1867         for (x = 0; x < 8; x++)
1868             score1 += (s1[x] - s2[x]) * (s1[x] - s2[x]);
1869         if (y + 1 < h) {
1870             for (x = 0; x < 7; x++)
1871                 score2 += FFABS(s1[x]     - s1[x + stride] -
1872                                 s1[x + 1] + s1[x + stride + 1]) -
1873                           FFABS(s2[x]     - s2[x + stride] -
1874                                 s2[x + 1] + s2[x + stride + 1]);
1875         }
1876         s1 += stride;
1877         s2 += stride;
1878     }
1879
1880     if (c)
1881         return score1 + FFABS(score2) * c->avctx->nsse_weight;
1882     else
1883         return score1 + FFABS(score2) * 8;
1884 }
1885
1886 static int try_8x8basis_c(int16_t rem[64], int16_t weight[64],
1887                           int16_t basis[64], int scale)
1888 {
1889     int i;
1890     unsigned int sum = 0;
1891
1892     for (i = 0; i < 8 * 8; i++) {
1893         int b = rem[i] + ((basis[i] * scale +
1894                            (1 << (BASIS_SHIFT - RECON_SHIFT - 1))) >>
1895                           (BASIS_SHIFT - RECON_SHIFT));
1896         int w = weight[i];
1897         b >>= RECON_SHIFT;
1898         assert(-512 < b && b < 512);
1899
1900         sum += (w * b) * (w * b) >> 4;
1901     }
1902     return sum >> 2;
1903 }
1904
1905 static void add_8x8basis_c(int16_t rem[64], int16_t basis[64], int scale)
1906 {
1907     int i;
1908
1909     for (i = 0; i < 8 * 8; i++)
1910         rem[i] += (basis[i] * scale +
1911                    (1 << (BASIS_SHIFT - RECON_SHIFT - 1))) >>
1912                   (BASIS_SHIFT - RECON_SHIFT);
1913 }
1914
1915 static int zero_cmp(void *s, uint8_t *a, uint8_t *b, int stride, int h)
1916 {
1917     return 0;
1918 }
1919
1920 void ff_set_cmp(DSPContext *c, me_cmp_func *cmp, int type)
1921 {
1922     int i;
1923
1924     memset(cmp, 0, sizeof(void *) * 6);
1925
1926     for (i = 0; i < 6; i++) {
1927         switch (type & 0xFF) {
1928         case FF_CMP_SAD:
1929             cmp[i] = c->sad[i];
1930             break;
1931         case FF_CMP_SATD:
1932             cmp[i] = c->hadamard8_diff[i];
1933             break;
1934         case FF_CMP_SSE:
1935             cmp[i] = c->sse[i];
1936             break;
1937         case FF_CMP_DCT:
1938             cmp[i] = c->dct_sad[i];
1939             break;
1940         case FF_CMP_DCT264:
1941             cmp[i] = c->dct264_sad[i];
1942             break;
1943         case FF_CMP_DCTMAX:
1944             cmp[i] = c->dct_max[i];
1945             break;
1946         case FF_CMP_PSNR:
1947             cmp[i] = c->quant_psnr[i];
1948             break;
1949         case FF_CMP_BIT:
1950             cmp[i] = c->bit[i];
1951             break;
1952         case FF_CMP_RD:
1953             cmp[i] = c->rd[i];
1954             break;
1955         case FF_CMP_VSAD:
1956             cmp[i] = c->vsad[i];
1957             break;
1958         case FF_CMP_VSSE:
1959             cmp[i] = c->vsse[i];
1960             break;
1961         case FF_CMP_ZERO:
1962             cmp[i] = zero_cmp;
1963             break;
1964         case FF_CMP_NSSE:
1965             cmp[i] = c->nsse[i];
1966             break;
1967         default:
1968             av_log(NULL, AV_LOG_ERROR,
1969                    "internal error in cmp function selection\n");
1970         }
1971     }
1972 }
1973
1974 static void add_bytes_c(uint8_t *dst, uint8_t *src, int w)
1975 {
1976     long i;
1977
1978     for (i = 0; i <= w - (int) sizeof(long); i += sizeof(long)) {
1979         long a = *(long *) (src + i);
1980         long b = *(long *) (dst + i);
1981         *(long *) (dst + i) = ((a & pb_7f) + (b & pb_7f)) ^ ((a ^ b) & pb_80);
1982     }
1983     for (; i < w; i++)
1984         dst[i + 0] += src[i + 0];
1985 }
1986
1987 static void diff_bytes_c(uint8_t *dst, uint8_t *src1, uint8_t *src2, int w)
1988 {
1989     long i;
1990
1991 #if !HAVE_FAST_UNALIGNED
1992     if ((long) src2 & (sizeof(long) - 1)) {
1993         for (i = 0; i + 7 < w; i += 8) {
1994             dst[i + 0] = src1[i + 0] - src2[i + 0];
1995             dst[i + 1] = src1[i + 1] - src2[i + 1];
1996             dst[i + 2] = src1[i + 2] - src2[i + 2];
1997             dst[i + 3] = src1[i + 3] - src2[i + 3];
1998             dst[i + 4] = src1[i + 4] - src2[i + 4];
1999             dst[i + 5] = src1[i + 5] - src2[i + 5];
2000             dst[i + 6] = src1[i + 6] - src2[i + 6];
2001             dst[i + 7] = src1[i + 7] - src2[i + 7];
2002         }
2003     } else
2004 #endif
2005     for (i = 0; i <= w - (int) sizeof(long); i += sizeof(long)) {
2006         long a = *(long *) (src1 + i);
2007         long b = *(long *) (src2 + i);
2008         *(long *) (dst + i) = ((a | pb_80) - (b & pb_7f)) ^
2009                               ((a ^ b ^ pb_80) & pb_80);
2010     }
2011     for (; i < w; i++)
2012         dst[i + 0] = src1[i + 0] - src2[i + 0];
2013 }
2014
2015 static void add_hfyu_median_prediction_c(uint8_t *dst, const uint8_t *src1,
2016                                          const uint8_t *diff, int w,
2017                                          int *left, int *left_top)
2018 {
2019     int i;
2020     uint8_t l, lt;
2021
2022     l  = *left;
2023     lt = *left_top;
2024
2025     for (i = 0; i < w; i++) {
2026         l      = mid_pred(l, src1[i], (l + src1[i] - lt) & 0xFF) + diff[i];
2027         lt     = src1[i];
2028         dst[i] = l;
2029     }
2030
2031     *left     = l;
2032     *left_top = lt;
2033 }
2034
2035 static void sub_hfyu_median_prediction_c(uint8_t *dst, const uint8_t *src1,
2036                                          const uint8_t *src2, int w,
2037                                          int *left, int *left_top)
2038 {
2039     int i;
2040     uint8_t l, lt;
2041
2042     l  = *left;
2043     lt = *left_top;
2044
2045     for (i = 0; i < w; i++) {
2046         const int pred = mid_pred(l, src1[i], (l + src1[i] - lt) & 0xFF);
2047         lt     = src1[i];
2048         l      = src2[i];
2049         dst[i] = l - pred;
2050     }
2051
2052     *left     = l;
2053     *left_top = lt;
2054 }
2055
2056 static int add_hfyu_left_prediction_c(uint8_t *dst, const uint8_t *src,
2057                                       int w, int acc)
2058 {
2059     int i;
2060
2061     for (i = 0; i < w - 1; i++) {
2062         acc   += src[i];
2063         dst[i] = acc;
2064         i++;
2065         acc   += src[i];
2066         dst[i] = acc;
2067     }
2068
2069     for (; i < w; i++) {
2070         acc   += src[i];
2071         dst[i] = acc;
2072     }
2073
2074     return acc;
2075 }
2076
2077 #if HAVE_BIGENDIAN
2078 #define B 3
2079 #define G 2
2080 #define R 1
2081 #define A 0
2082 #else
2083 #define B 0
2084 #define G 1
2085 #define R 2
2086 #define A 3
2087 #endif
2088 static void add_hfyu_left_prediction_bgr32_c(uint8_t *dst, const uint8_t *src,
2089                                              int w, int *red, int *green,
2090                                              int *blue, int *alpha)
2091 {
2092     int i, r = *red, g = *green, b = *blue, a = *alpha;
2093
2094     for (i = 0; i < w; i++) {
2095         b += src[4 * i + B];
2096         g += src[4 * i + G];
2097         r += src[4 * i + R];
2098         a += src[4 * i + A];
2099
2100         dst[4 * i + B] = b;
2101         dst[4 * i + G] = g;
2102         dst[4 * i + R] = r;
2103         dst[4 * i + A] = a;
2104     }
2105
2106     *red   = r;
2107     *green = g;
2108     *blue  = b;
2109     *alpha = a;
2110 }
2111 #undef B
2112 #undef G
2113 #undef R
2114 #undef A
2115
2116 #define BUTTERFLY2(o1, o2, i1, i2)              \
2117     o1 = (i1) + (i2);                           \
2118     o2 = (i1) - (i2);
2119
2120 #define BUTTERFLY1(x, y)                        \
2121     {                                           \
2122         int a, b;                               \
2123         a = x;                                  \
2124         b = y;                                  \
2125         x = a + b;                              \
2126         y = a - b;                              \
2127     }
2128
2129 #define BUTTERFLYA(x, y) (FFABS((x) + (y)) + FFABS((x) - (y)))
2130
2131 static int hadamard8_diff8x8_c(/* MpegEncContext */ void *s, uint8_t *dst,
2132                                uint8_t *src, int stride, int h)
2133 {
2134     int i, temp[64], sum = 0;
2135
2136     assert(h == 8);
2137
2138     for (i = 0; i < 8; i++) {
2139         // FIXME: try pointer walks
2140         BUTTERFLY2(temp[8 * i + 0], temp[8 * i + 1],
2141                    src[stride * i + 0] - dst[stride * i + 0],
2142                    src[stride * i + 1] - dst[stride * i + 1]);
2143         BUTTERFLY2(temp[8 * i + 2], temp[8 * i + 3],
2144                    src[stride * i + 2] - dst[stride * i + 2],
2145                    src[stride * i + 3] - dst[stride * i + 3]);
2146         BUTTERFLY2(temp[8 * i + 4], temp[8 * i + 5],
2147                    src[stride * i + 4] - dst[stride * i + 4],
2148                    src[stride * i + 5] - dst[stride * i + 5]);
2149         BUTTERFLY2(temp[8 * i + 6], temp[8 * i + 7],
2150                    src[stride * i + 6] - dst[stride * i + 6],
2151                    src[stride * i + 7] - dst[stride * i + 7]);
2152
2153         BUTTERFLY1(temp[8 * i + 0], temp[8 * i + 2]);
2154         BUTTERFLY1(temp[8 * i + 1], temp[8 * i + 3]);
2155         BUTTERFLY1(temp[8 * i + 4], temp[8 * i + 6]);
2156         BUTTERFLY1(temp[8 * i + 5], temp[8 * i + 7]);
2157
2158         BUTTERFLY1(temp[8 * i + 0], temp[8 * i + 4]);
2159         BUTTERFLY1(temp[8 * i + 1], temp[8 * i + 5]);
2160         BUTTERFLY1(temp[8 * i + 2], temp[8 * i + 6]);
2161         BUTTERFLY1(temp[8 * i + 3], temp[8 * i + 7]);
2162     }
2163
2164     for (i = 0; i < 8; i++) {
2165         BUTTERFLY1(temp[8 * 0 + i], temp[8 * 1 + i]);
2166         BUTTERFLY1(temp[8 * 2 + i], temp[8 * 3 + i]);
2167         BUTTERFLY1(temp[8 * 4 + i], temp[8 * 5 + i]);
2168         BUTTERFLY1(temp[8 * 6 + i], temp[8 * 7 + i]);
2169
2170         BUTTERFLY1(temp[8 * 0 + i], temp[8 * 2 + i]);
2171         BUTTERFLY1(temp[8 * 1 + i], temp[8 * 3 + i]);
2172         BUTTERFLY1(temp[8 * 4 + i], temp[8 * 6 + i]);
2173         BUTTERFLY1(temp[8 * 5 + i], temp[8 * 7 + i]);
2174
2175         sum += BUTTERFLYA(temp[8 * 0 + i], temp[8 * 4 + i]) +
2176                BUTTERFLYA(temp[8 * 1 + i], temp[8 * 5 + i]) +
2177                BUTTERFLYA(temp[8 * 2 + i], temp[8 * 6 + i]) +
2178                BUTTERFLYA(temp[8 * 3 + i], temp[8 * 7 + i]);
2179     }
2180     return sum;
2181 }
2182
2183 static int hadamard8_intra8x8_c(/* MpegEncContext */ void *s, uint8_t *src,
2184                                 uint8_t *dummy, int stride, int h)
2185 {
2186     int i, temp[64], sum = 0;
2187
2188     assert(h == 8);
2189
2190     for (i = 0; i < 8; i++) {
2191         // FIXME: try pointer walks
2192         BUTTERFLY2(temp[8 * i + 0], temp[8 * i + 1],
2193                    src[stride * i + 0], src[stride * i + 1]);
2194         BUTTERFLY2(temp[8 * i + 2], temp[8 * i + 3],
2195                    src[stride * i + 2], src[stride * i + 3]);
2196         BUTTERFLY2(temp[8 * i + 4], temp[8 * i + 5],
2197                    src[stride * i + 4], src[stride * i + 5]);
2198         BUTTERFLY2(temp[8 * i + 6], temp[8 * i + 7],
2199                    src[stride * i + 6], src[stride * i + 7]);
2200
2201         BUTTERFLY1(temp[8 * i + 0], temp[8 * i + 2]);
2202         BUTTERFLY1(temp[8 * i + 1], temp[8 * i + 3]);
2203         BUTTERFLY1(temp[8 * i + 4], temp[8 * i + 6]);
2204         BUTTERFLY1(temp[8 * i + 5], temp[8 * i + 7]);
2205
2206         BUTTERFLY1(temp[8 * i + 0], temp[8 * i + 4]);
2207         BUTTERFLY1(temp[8 * i + 1], temp[8 * i + 5]);
2208         BUTTERFLY1(temp[8 * i + 2], temp[8 * i + 6]);
2209         BUTTERFLY1(temp[8 * i + 3], temp[8 * i + 7]);
2210     }
2211
2212     for (i = 0; i < 8; i++) {
2213         BUTTERFLY1(temp[8 * 0 + i], temp[8 * 1 + i]);
2214         BUTTERFLY1(temp[8 * 2 + i], temp[8 * 3 + i]);
2215         BUTTERFLY1(temp[8 * 4 + i], temp[8 * 5 + i]);
2216         BUTTERFLY1(temp[8 * 6 + i], temp[8 * 7 + i]);
2217
2218         BUTTERFLY1(temp[8 * 0 + i], temp[8 * 2 + i]);
2219         BUTTERFLY1(temp[8 * 1 + i], temp[8 * 3 + i]);
2220         BUTTERFLY1(temp[8 * 4 + i], temp[8 * 6 + i]);
2221         BUTTERFLY1(temp[8 * 5 + i], temp[8 * 7 + i]);
2222
2223         sum +=
2224             BUTTERFLYA(temp[8 * 0 + i], temp[8 * 4 + i])
2225             + BUTTERFLYA(temp[8 * 1 + i], temp[8 * 5 + i])
2226             + BUTTERFLYA(temp[8 * 2 + i], temp[8 * 6 + i])
2227             + BUTTERFLYA(temp[8 * 3 + i], temp[8 * 7 + i]);
2228     }
2229
2230     sum -= FFABS(temp[8 * 0] + temp[8 * 4]); // -mean
2231
2232     return sum;
2233 }
2234
2235 static int dct_sad8x8_c(/* MpegEncContext */ void *c, uint8_t *src1,
2236                         uint8_t *src2, int stride, int h)
2237 {
2238     MpegEncContext *const s = (MpegEncContext *) c;
2239     LOCAL_ALIGNED_16(int16_t, temp, [64]);
2240
2241     assert(h == 8);
2242
2243     s->dsp.diff_pixels(temp, src1, src2, stride);
2244     s->dsp.fdct(temp);
2245     return s->dsp.sum_abs_dctelem(temp);
2246 }
2247
2248 #if CONFIG_GPL
2249 #define DCT8_1D                                         \
2250     {                                                   \
2251         const int s07 = SRC(0) + SRC(7);                \
2252         const int s16 = SRC(1) + SRC(6);                \
2253         const int s25 = SRC(2) + SRC(5);                \
2254         const int s34 = SRC(3) + SRC(4);                \
2255         const int a0  = s07 + s34;                      \
2256         const int a1  = s16 + s25;                      \
2257         const int a2  = s07 - s34;                      \
2258         const int a3  = s16 - s25;                      \
2259         const int d07 = SRC(0) - SRC(7);                \
2260         const int d16 = SRC(1) - SRC(6);                \
2261         const int d25 = SRC(2) - SRC(5);                \
2262         const int d34 = SRC(3) - SRC(4);                \
2263         const int a4  = d16 + d25 + (d07 + (d07 >> 1)); \
2264         const int a5  = d07 - d34 - (d25 + (d25 >> 1)); \
2265         const int a6  = d07 + d34 - (d16 + (d16 >> 1)); \
2266         const int a7  = d16 - d25 + (d34 + (d34 >> 1)); \
2267         DST(0, a0 + a1);                                \
2268         DST(1, a4 + (a7 >> 2));                         \
2269         DST(2, a2 + (a3 >> 1));                         \
2270         DST(3, a5 + (a6 >> 2));                         \
2271         DST(4, a0 - a1);                                \
2272         DST(5, a6 - (a5 >> 2));                         \
2273         DST(6, (a2 >> 1) - a3);                         \
2274         DST(7, (a4 >> 2) - a7);                         \
2275     }
2276
2277 static int dct264_sad8x8_c(/* MpegEncContext */ void *c, uint8_t *src1,
2278                            uint8_t *src2, int stride, int h)
2279 {
2280     MpegEncContext *const s = (MpegEncContext *) c;
2281     int16_t dct[8][8];
2282     int i, sum = 0;
2283
2284     s->dsp.diff_pixels(dct[0], src1, src2, stride);
2285
2286 #define SRC(x) dct[i][x]
2287 #define DST(x, v) dct[i][x] = v
2288     for (i = 0; i < 8; i++)
2289         DCT8_1D
2290 #undef SRC
2291 #undef DST
2292
2293 #define SRC(x) dct[x][i]
2294 #define DST(x, v) sum += FFABS(v)
2295         for (i = 0; i < 8; i++)
2296             DCT8_1D
2297 #undef SRC
2298 #undef DST
2299             return sum;
2300 }
2301 #endif
2302
2303 static int dct_max8x8_c(/* MpegEncContext */ void *c, uint8_t *src1,
2304                         uint8_t *src2, int stride, int h)
2305 {
2306     MpegEncContext *const s = (MpegEncContext *) c;
2307     LOCAL_ALIGNED_16(int16_t, temp, [64]);
2308     int sum = 0, i;
2309
2310     assert(h == 8);
2311
2312     s->dsp.diff_pixels(temp, src1, src2, stride);
2313     s->dsp.fdct(temp);
2314
2315     for (i = 0; i < 64; i++)
2316         sum = FFMAX(sum, FFABS(temp[i]));
2317
2318     return sum;
2319 }
2320
2321 static int quant_psnr8x8_c(/* MpegEncContext */ void *c, uint8_t *src1,
2322                            uint8_t *src2, int stride, int h)
2323 {
2324     MpegEncContext *const s = c;
2325     LOCAL_ALIGNED_16(int16_t, temp, [64 * 2]);
2326     int16_t *const bak = temp + 64;
2327     int sum = 0, i;
2328
2329     assert(h == 8);
2330     s->mb_intra = 0;
2331
2332     s->dsp.diff_pixels(temp, src1, src2, stride);
2333
2334     memcpy(bak, temp, 64 * sizeof(int16_t));
2335
2336     s->block_last_index[0 /* FIXME */] =
2337         s->fast_dct_quantize(s, temp, 0 /* FIXME */, s->qscale, &i);
2338     s->dct_unquantize_inter(s, temp, 0, s->qscale);
2339     ff_simple_idct_8(temp); // FIXME
2340
2341     for (i = 0; i < 64; i++)
2342         sum += (temp[i] - bak[i]) * (temp[i] - bak[i]);
2343
2344     return sum;
2345 }
2346
2347 static int rd8x8_c(/* MpegEncContext */ void *c, uint8_t *src1, uint8_t *src2,
2348                    int stride, int h)
2349 {
2350     MpegEncContext *const s  = (MpegEncContext *) c;
2351     const uint8_t *scantable = s->intra_scantable.permutated;
2352     LOCAL_ALIGNED_16(int16_t, temp, [64]);
2353     LOCAL_ALIGNED_16(uint8_t, lsrc1, [64]);
2354     LOCAL_ALIGNED_16(uint8_t, lsrc2, [64]);
2355     int i, last, run, bits, level, distortion, start_i;
2356     const int esc_length = s->ac_esc_length;
2357     uint8_t *length, *last_length;
2358
2359     assert(h == 8);
2360
2361     copy_block8(lsrc1, src1, 8, stride, 8);
2362     copy_block8(lsrc2, src2, 8, stride, 8);
2363
2364     s->dsp.diff_pixels(temp, lsrc1, lsrc2, 8);
2365
2366     s->block_last_index[0 /* FIXME */] =
2367     last                               =
2368         s->fast_dct_quantize(s, temp, 0 /* FIXME */, s->qscale, &i);
2369
2370     bits = 0;
2371
2372     if (s->mb_intra) {
2373         start_i     = 1;
2374         length      = s->intra_ac_vlc_length;
2375         last_length = s->intra_ac_vlc_last_length;
2376         bits       += s->luma_dc_vlc_length[temp[0] + 256]; // FIXME: chroma
2377     } else {
2378         start_i     = 0;
2379         length      = s->inter_ac_vlc_length;
2380         last_length = s->inter_ac_vlc_last_length;
2381     }
2382
2383     if (last >= start_i) {
2384         run = 0;
2385         for (i = start_i; i < last; i++) {
2386             int j = scantable[i];
2387             level = temp[j];
2388
2389             if (level) {
2390                 level += 64;
2391                 if ((level & (~127)) == 0)
2392                     bits += length[UNI_AC_ENC_INDEX(run, level)];
2393                 else
2394                     bits += esc_length;
2395                 run = 0;
2396             } else
2397                 run++;
2398         }
2399         i = scantable[last];
2400
2401         level = temp[i] + 64;
2402
2403         assert(level - 64);
2404
2405         if ((level & (~127)) == 0) {
2406             bits += last_length[UNI_AC_ENC_INDEX(run, level)];
2407         } else
2408             bits += esc_length;
2409     }
2410
2411     if (last >= 0) {
2412         if (s->mb_intra)
2413             s->dct_unquantize_intra(s, temp, 0, s->qscale);
2414         else
2415             s->dct_unquantize_inter(s, temp, 0, s->qscale);
2416     }
2417
2418     s->dsp.idct_add(lsrc2, 8, temp);
2419
2420     distortion = s->dsp.sse[1](NULL, lsrc2, lsrc1, 8, 8);
2421
2422     return distortion + ((bits * s->qscale * s->qscale * 109 + 64) >> 7);
2423 }
2424
2425 static int bit8x8_c(/* MpegEncContext */ void *c, uint8_t *src1, uint8_t *src2,
2426                     int stride, int h)
2427 {
2428     MpegEncContext *const s  = (MpegEncContext *) c;
2429     const uint8_t *scantable = s->intra_scantable.permutated;
2430     LOCAL_ALIGNED_16(int16_t, temp, [64]);
2431     int i, last, run, bits, level, start_i;
2432     const int esc_length = s->ac_esc_length;
2433     uint8_t *length, *last_length;
2434
2435     assert(h == 8);
2436
2437     s->dsp.diff_pixels(temp, src1, src2, stride);
2438
2439     s->block_last_index[0 /* FIXME */] =
2440     last                               =
2441         s->fast_dct_quantize(s, temp, 0 /* FIXME */, s->qscale, &i);
2442
2443     bits = 0;
2444
2445     if (s->mb_intra) {
2446         start_i     = 1;
2447         length      = s->intra_ac_vlc_length;
2448         last_length = s->intra_ac_vlc_last_length;
2449         bits       += s->luma_dc_vlc_length[temp[0] + 256]; // FIXME: chroma
2450     } else {
2451         start_i     = 0;
2452         length      = s->inter_ac_vlc_length;
2453         last_length = s->inter_ac_vlc_last_length;
2454     }
2455
2456     if (last >= start_i) {
2457         run = 0;
2458         for (i = start_i; i < last; i++) {
2459             int j = scantable[i];
2460             level = temp[j];
2461
2462             if (level) {
2463                 level += 64;
2464                 if ((level & (~127)) == 0)
2465                     bits += length[UNI_AC_ENC_INDEX(run, level)];
2466                 else
2467                     bits += esc_length;
2468                 run = 0;
2469             } else
2470                 run++;
2471         }
2472         i = scantable[last];
2473
2474         level = temp[i] + 64;
2475
2476         assert(level - 64);
2477
2478         if ((level & (~127)) == 0)
2479             bits += last_length[UNI_AC_ENC_INDEX(run, level)];
2480         else
2481             bits += esc_length;
2482     }
2483
2484     return bits;
2485 }
2486
2487 #define VSAD_INTRA(size)                                                \
2488 static int vsad_intra ## size ## _c(/* MpegEncContext */ void *c,       \
2489                                     uint8_t *s, uint8_t *dummy,         \
2490                                     int stride, int h)                  \
2491 {                                                                       \
2492     int score = 0, x, y;                                                \
2493                                                                         \
2494     for (y = 1; y < h; y++) {                                           \
2495         for (x = 0; x < size; x += 4) {                                 \
2496             score += FFABS(s[x]     - s[x + stride])     +              \
2497                      FFABS(s[x + 1] - s[x + stride + 1]) +              \
2498                      FFABS(s[x + 2] - s[x + 2 + stride]) +              \
2499                      FFABS(s[x + 3] - s[x + 3 + stride]);               \
2500         }                                                               \
2501         s += stride;                                                    \
2502     }                                                                   \
2503                                                                         \
2504     return score;                                                       \
2505 }
2506 VSAD_INTRA(8)
2507 VSAD_INTRA(16)
2508
2509 static int vsad16_c(/* MpegEncContext */ void *c, uint8_t *s1, uint8_t *s2,
2510                     int stride, int h)
2511 {
2512     int score = 0, x, y;
2513
2514     for (y = 1; y < h; y++) {
2515         for (x = 0; x < 16; x++)
2516             score += FFABS(s1[x] - s2[x] - s1[x + stride] + s2[x + stride]);
2517         s1 += stride;
2518         s2 += stride;
2519     }
2520
2521     return score;
2522 }
2523
2524 #define SQ(a) ((a) * (a))
2525 #define VSSE_INTRA(size)                                                \
2526 static int vsse_intra ## size ## _c(/* MpegEncContext */ void *c,       \
2527                                     uint8_t *s, uint8_t *dummy,         \
2528                                     int stride, int h)                  \
2529 {                                                                       \
2530     int score = 0, x, y;                                                \
2531                                                                         \
2532     for (y = 1; y < h; y++) {                                           \
2533         for (x = 0; x < size; x += 4) {                                 \
2534             score += SQ(s[x]     - s[x + stride]) +                     \
2535                      SQ(s[x + 1] - s[x + stride + 1]) +                 \
2536                      SQ(s[x + 2] - s[x + stride + 2]) +                 \
2537                      SQ(s[x + 3] - s[x + stride + 3]);                  \
2538         }                                                               \
2539         s += stride;                                                    \
2540     }                                                                   \
2541                                                                         \
2542     return score;                                                       \
2543 }
2544 VSSE_INTRA(8)
2545 VSSE_INTRA(16)
2546
2547 static int vsse16_c(/* MpegEncContext */ void *c, uint8_t *s1, uint8_t *s2,
2548                     int stride, int h)
2549 {
2550     int score = 0, x, y;
2551
2552     for (y = 1; y < h; y++) {
2553         for (x = 0; x < 16; x++)
2554             score += SQ(s1[x] - s2[x] - s1[x + stride] + s2[x + stride]);
2555         s1 += stride;
2556         s2 += stride;
2557     }
2558
2559     return score;
2560 }
2561
2562 static int ssd_int8_vs_int16_c(const int8_t *pix1, const int16_t *pix2,
2563                                int size)
2564 {
2565     int score = 0, i;
2566
2567     for (i = 0; i < size; i++)
2568         score += (pix1[i] - pix2[i]) * (pix1[i] - pix2[i]);
2569     return score;
2570 }
2571
2572 #define WRAPPER8_16_SQ(name8, name16)                                   \
2573 static int name16(void /*MpegEncContext*/ *s,                           \
2574                   uint8_t *dst, uint8_t *src,                           \
2575                   int stride, int h)                                    \
2576 {                                                                       \
2577     int score = 0;                                                      \
2578                                                                         \
2579     score += name8(s, dst, src, stride, 8);                             \
2580     score += name8(s, dst + 8, src + 8, stride, 8);                     \
2581     if (h == 16) {                                                      \
2582         dst   += 8 * stride;                                            \
2583         src   += 8 * stride;                                            \
2584         score += name8(s, dst, src, stride, 8);                         \
2585         score += name8(s, dst + 8, src + 8, stride, 8);                 \
2586     }                                                                   \
2587     return score;                                                       \
2588 }
2589
2590 WRAPPER8_16_SQ(hadamard8_diff8x8_c, hadamard8_diff16_c)
2591 WRAPPER8_16_SQ(hadamard8_intra8x8_c, hadamard8_intra16_c)
2592 WRAPPER8_16_SQ(dct_sad8x8_c, dct_sad16_c)
2593 #if CONFIG_GPL
2594 WRAPPER8_16_SQ(dct264_sad8x8_c, dct264_sad16_c)
2595 #endif
2596 WRAPPER8_16_SQ(dct_max8x8_c, dct_max16_c)
2597 WRAPPER8_16_SQ(quant_psnr8x8_c, quant_psnr16_c)
2598 WRAPPER8_16_SQ(rd8x8_c, rd16_c)
2599 WRAPPER8_16_SQ(bit8x8_c, bit16_c)
2600
2601 static inline uint32_t clipf_c_one(uint32_t a, uint32_t mini,
2602                                    uint32_t maxi, uint32_t maxisign)
2603 {
2604     if (a > mini)
2605         return mini;
2606     else if ((a ^ (1U << 31)) > maxisign)
2607         return maxi;
2608     else
2609         return a;
2610 }
2611
2612 static void vector_clipf_c_opposite_sign(float *dst, const float *src,
2613                                          float *min, float *max, int len)
2614 {
2615     int i;
2616     uint32_t mini        = *(uint32_t *) min;
2617     uint32_t maxi        = *(uint32_t *) max;
2618     uint32_t maxisign    = maxi ^ (1U << 31);
2619     uint32_t *dsti       = (uint32_t *) dst;
2620     const uint32_t *srci = (const uint32_t *) src;
2621
2622     for (i = 0; i < len; i += 8) {
2623         dsti[i + 0] = clipf_c_one(srci[i + 0], mini, maxi, maxisign);
2624         dsti[i + 1] = clipf_c_one(srci[i + 1], mini, maxi, maxisign);
2625         dsti[i + 2] = clipf_c_one(srci[i + 2], mini, maxi, maxisign);
2626         dsti[i + 3] = clipf_c_one(srci[i + 3], mini, maxi, maxisign);
2627         dsti[i + 4] = clipf_c_one(srci[i + 4], mini, maxi, maxisign);
2628         dsti[i + 5] = clipf_c_one(srci[i + 5], mini, maxi, maxisign);
2629         dsti[i + 6] = clipf_c_one(srci[i + 6], mini, maxi, maxisign);
2630         dsti[i + 7] = clipf_c_one(srci[i + 7], mini, maxi, maxisign);
2631     }
2632 }
2633
2634 static void vector_clipf_c(float *dst, const float *src,
2635                            float min, float max, int len)
2636 {
2637     int i;
2638
2639     if (min < 0 && max > 0) {
2640         vector_clipf_c_opposite_sign(dst, src, &min, &max, len);
2641     } else {
2642         for (i = 0; i < len; i += 8) {
2643             dst[i]     = av_clipf(src[i], min, max);
2644             dst[i + 1] = av_clipf(src[i + 1], min, max);
2645             dst[i + 2] = av_clipf(src[i + 2], min, max);
2646             dst[i + 3] = av_clipf(src[i + 3], min, max);
2647             dst[i + 4] = av_clipf(src[i + 4], min, max);
2648             dst[i + 5] = av_clipf(src[i + 5], min, max);
2649             dst[i + 6] = av_clipf(src[i + 6], min, max);
2650             dst[i + 7] = av_clipf(src[i + 7], min, max);
2651         }
2652     }
2653 }
2654
2655 static int32_t scalarproduct_int16_c(const int16_t *v1, const int16_t *v2,
2656                                      int order)
2657 {
2658     int res = 0;
2659
2660     while (order--)
2661         res += *v1++ **v2++;
2662
2663     return res;
2664 }
2665
2666 static int32_t scalarproduct_and_madd_int16_c(int16_t *v1, const int16_t *v2,
2667                                               const int16_t *v3,
2668                                               int order, int mul)
2669 {
2670     int res = 0;
2671
2672     while (order--) {
2673         res   += *v1 * *v2++;
2674         *v1++ += mul * *v3++;
2675     }
2676     return res;
2677 }
2678
2679 static void vector_clip_int32_c(int32_t *dst, const int32_t *src, int32_t min,
2680                                 int32_t max, unsigned int len)
2681 {
2682     do {
2683         *dst++ = av_clip(*src++, min, max);
2684         *dst++ = av_clip(*src++, min, max);
2685         *dst++ = av_clip(*src++, min, max);
2686         *dst++ = av_clip(*src++, min, max);
2687         *dst++ = av_clip(*src++, min, max);
2688         *dst++ = av_clip(*src++, min, max);
2689         *dst++ = av_clip(*src++, min, max);
2690         *dst++ = av_clip(*src++, min, max);
2691         len   -= 8;
2692     } while (len > 0);
2693 }
2694
2695 static void jref_idct_put(uint8_t *dest, int line_size, int16_t *block)
2696 {
2697     ff_j_rev_dct(block);
2698     put_pixels_clamped_c(block, dest, line_size);
2699 }
2700
2701 static void jref_idct_add(uint8_t *dest, int line_size, int16_t *block)
2702 {
2703     ff_j_rev_dct(block);
2704     add_pixels_clamped_c(block, dest, line_size);
2705 }
2706
2707 /* init static data */
2708 av_cold void ff_dsputil_static_init(void)
2709 {
2710     int i;
2711
2712     for (i = 0; i < 512; i++)
2713         ff_square_tab[i] = (i - 256) * (i - 256);
2714 }
2715
2716 av_cold void ff_dsputil_init(DSPContext *c, AVCodecContext *avctx)
2717 {
2718     const unsigned high_bit_depth = avctx->bits_per_raw_sample > 8;
2719
2720 #if CONFIG_ENCODERS
2721     if (avctx->bits_per_raw_sample == 10) {
2722         c->fdct    = ff_jpeg_fdct_islow_10;
2723         c->fdct248 = ff_fdct248_islow_10;
2724     } else {
2725         if (avctx->dct_algo == FF_DCT_FASTINT) {
2726             c->fdct    = ff_fdct_ifast;
2727             c->fdct248 = ff_fdct_ifast248;
2728         } else if (avctx->dct_algo == FF_DCT_FAAN) {
2729             c->fdct    = ff_faandct;
2730             c->fdct248 = ff_faandct248;
2731         } else {
2732             c->fdct    = ff_jpeg_fdct_islow_8; // slow/accurate/default
2733             c->fdct248 = ff_fdct248_islow_8;
2734         }
2735     }
2736 #endif /* CONFIG_ENCODERS */
2737
2738     if (avctx->bits_per_raw_sample == 10) {
2739         c->idct_put              = ff_simple_idct_put_10;
2740         c->idct_add              = ff_simple_idct_add_10;
2741         c->idct                  = ff_simple_idct_10;
2742         c->idct_permutation_type = FF_NO_IDCT_PERM;
2743     } else {
2744         if (avctx->idct_algo == FF_IDCT_INT) {
2745             c->idct_put              = jref_idct_put;
2746             c->idct_add              = jref_idct_add;
2747             c->idct                  = ff_j_rev_dct;
2748             c->idct_permutation_type = FF_LIBMPEG2_IDCT_PERM;
2749         } else if (avctx->idct_algo == FF_IDCT_FAAN) {
2750             c->idct_put              = ff_faanidct_put;
2751             c->idct_add              = ff_faanidct_add;
2752             c->idct                  = ff_faanidct;
2753             c->idct_permutation_type = FF_NO_IDCT_PERM;
2754         } else { // accurate/default
2755             c->idct_put              = ff_simple_idct_put_8;
2756             c->idct_add              = ff_simple_idct_add_8;
2757             c->idct                  = ff_simple_idct_8;
2758             c->idct_permutation_type = FF_NO_IDCT_PERM;
2759         }
2760     }
2761
2762     c->diff_pixels = diff_pixels_c;
2763
2764     c->put_pixels_clamped        = put_pixels_clamped_c;
2765     c->put_signed_pixels_clamped = put_signed_pixels_clamped_c;
2766     c->add_pixels_clamped        = add_pixels_clamped_c;
2767
2768     c->sum_abs_dctelem = sum_abs_dctelem_c;
2769
2770     c->gmc1 = gmc1_c;
2771     c->gmc  = ff_gmc_c;
2772
2773     c->pix_sum   = pix_sum_c;
2774     c->pix_norm1 = pix_norm1_c;
2775
2776     c->fill_block_tab[0] = fill_block16_c;
2777     c->fill_block_tab[1] = fill_block8_c;
2778
2779     /* TODO [0] 16  [1] 8 */
2780     c->pix_abs[0][0] = pix_abs16_c;
2781     c->pix_abs[0][1] = pix_abs16_x2_c;
2782     c->pix_abs[0][2] = pix_abs16_y2_c;
2783     c->pix_abs[0][3] = pix_abs16_xy2_c;
2784     c->pix_abs[1][0] = pix_abs8_c;
2785     c->pix_abs[1][1] = pix_abs8_x2_c;
2786     c->pix_abs[1][2] = pix_abs8_y2_c;
2787     c->pix_abs[1][3] = pix_abs8_xy2_c;
2788
2789     c->put_tpel_pixels_tab[0]  = put_tpel_pixels_mc00_c;
2790     c->put_tpel_pixels_tab[1]  = put_tpel_pixels_mc10_c;
2791     c->put_tpel_pixels_tab[2]  = put_tpel_pixels_mc20_c;
2792     c->put_tpel_pixels_tab[4]  = put_tpel_pixels_mc01_c;
2793     c->put_tpel_pixels_tab[5]  = put_tpel_pixels_mc11_c;
2794     c->put_tpel_pixels_tab[6]  = put_tpel_pixels_mc21_c;
2795     c->put_tpel_pixels_tab[8]  = put_tpel_pixels_mc02_c;
2796     c->put_tpel_pixels_tab[9]  = put_tpel_pixels_mc12_c;
2797     c->put_tpel_pixels_tab[10] = put_tpel_pixels_mc22_c;
2798
2799     c->avg_tpel_pixels_tab[0]  = avg_tpel_pixels_mc00_c;
2800     c->avg_tpel_pixels_tab[1]  = avg_tpel_pixels_mc10_c;
2801     c->avg_tpel_pixels_tab[2]  = avg_tpel_pixels_mc20_c;
2802     c->avg_tpel_pixels_tab[4]  = avg_tpel_pixels_mc01_c;
2803     c->avg_tpel_pixels_tab[5]  = avg_tpel_pixels_mc11_c;
2804     c->avg_tpel_pixels_tab[6]  = avg_tpel_pixels_mc21_c;
2805     c->avg_tpel_pixels_tab[8]  = avg_tpel_pixels_mc02_c;
2806     c->avg_tpel_pixels_tab[9]  = avg_tpel_pixels_mc12_c;
2807     c->avg_tpel_pixels_tab[10] = avg_tpel_pixels_mc22_c;
2808
2809 #define dspfunc(PFX, IDX, NUM)                              \
2810     c->PFX ## _pixels_tab[IDX][0]  = PFX ## NUM ## _mc00_c; \
2811     c->PFX ## _pixels_tab[IDX][1]  = PFX ## NUM ## _mc10_c; \
2812     c->PFX ## _pixels_tab[IDX][2]  = PFX ## NUM ## _mc20_c; \
2813     c->PFX ## _pixels_tab[IDX][3]  = PFX ## NUM ## _mc30_c; \
2814     c->PFX ## _pixels_tab[IDX][4]  = PFX ## NUM ## _mc01_c; \
2815     c->PFX ## _pixels_tab[IDX][5]  = PFX ## NUM ## _mc11_c; \
2816     c->PFX ## _pixels_tab[IDX][6]  = PFX ## NUM ## _mc21_c; \
2817     c->PFX ## _pixels_tab[IDX][7]  = PFX ## NUM ## _mc31_c; \
2818     c->PFX ## _pixels_tab[IDX][8]  = PFX ## NUM ## _mc02_c; \
2819     c->PFX ## _pixels_tab[IDX][9]  = PFX ## NUM ## _mc12_c; \
2820     c->PFX ## _pixels_tab[IDX][10] = PFX ## NUM ## _mc22_c; \
2821     c->PFX ## _pixels_tab[IDX][11] = PFX ## NUM ## _mc32_c; \
2822     c->PFX ## _pixels_tab[IDX][12] = PFX ## NUM ## _mc03_c; \
2823     c->PFX ## _pixels_tab[IDX][13] = PFX ## NUM ## _mc13_c; \
2824     c->PFX ## _pixels_tab[IDX][14] = PFX ## NUM ## _mc23_c; \
2825     c->PFX ## _pixels_tab[IDX][15] = PFX ## NUM ## _mc33_c
2826
2827     dspfunc(put_qpel, 0, 16);
2828     dspfunc(put_qpel, 1, 8);
2829
2830     dspfunc(put_no_rnd_qpel, 0, 16);
2831     dspfunc(put_no_rnd_qpel, 1, 8);
2832
2833     dspfunc(avg_qpel, 0, 16);
2834     dspfunc(avg_qpel, 1, 8);
2835
2836 #undef dspfunc
2837
2838     c->put_mspel_pixels_tab[0] = ff_put_pixels8x8_c;
2839     c->put_mspel_pixels_tab[1] = put_mspel8_mc10_c;
2840     c->put_mspel_pixels_tab[2] = put_mspel8_mc20_c;
2841     c->put_mspel_pixels_tab[3] = put_mspel8_mc30_c;
2842     c->put_mspel_pixels_tab[4] = put_mspel8_mc02_c;
2843     c->put_mspel_pixels_tab[5] = put_mspel8_mc12_c;
2844     c->put_mspel_pixels_tab[6] = put_mspel8_mc22_c;
2845     c->put_mspel_pixels_tab[7] = put_mspel8_mc32_c;
2846
2847 #define SET_CMP_FUNC(name)                      \
2848     c->name[0] = name ## 16_c;                  \
2849     c->name[1] = name ## 8x8_c;
2850
2851     SET_CMP_FUNC(hadamard8_diff)
2852     c->hadamard8_diff[4] = hadamard8_intra16_c;
2853     c->hadamard8_diff[5] = hadamard8_intra8x8_c;
2854     SET_CMP_FUNC(dct_sad)
2855     SET_CMP_FUNC(dct_max)
2856 #if CONFIG_GPL
2857     SET_CMP_FUNC(dct264_sad)
2858 #endif
2859     c->sad[0] = pix_abs16_c;
2860     c->sad[1] = pix_abs8_c;
2861     c->sse[0] = sse16_c;
2862     c->sse[1] = sse8_c;
2863     c->sse[2] = sse4_c;
2864     SET_CMP_FUNC(quant_psnr)
2865     SET_CMP_FUNC(rd)
2866     SET_CMP_FUNC(bit)
2867     c->vsad[0] = vsad16_c;
2868     c->vsad[4] = vsad_intra16_c;
2869     c->vsad[5] = vsad_intra8_c;
2870     c->vsse[0] = vsse16_c;
2871     c->vsse[4] = vsse_intra16_c;
2872     c->vsse[5] = vsse_intra8_c;
2873     c->nsse[0] = nsse16_c;
2874     c->nsse[1] = nsse8_c;
2875
2876     c->ssd_int8_vs_int16 = ssd_int8_vs_int16_c;
2877
2878     c->add_bytes                      = add_bytes_c;
2879     c->add_hfyu_median_prediction     = add_hfyu_median_prediction_c;
2880     c->add_hfyu_left_prediction       = add_hfyu_left_prediction_c;
2881     c->add_hfyu_left_prediction_bgr32 = add_hfyu_left_prediction_bgr32_c;
2882
2883     c->diff_bytes                 = diff_bytes_c;
2884     c->sub_hfyu_median_prediction = sub_hfyu_median_prediction_c;
2885
2886     c->bswap_buf   = bswap_buf;
2887     c->bswap16_buf = bswap16_buf;
2888
2889     c->try_8x8basis = try_8x8basis_c;
2890     c->add_8x8basis = add_8x8basis_c;
2891
2892     c->scalarproduct_and_madd_int16 = scalarproduct_and_madd_int16_c;
2893
2894     c->scalarproduct_int16 = scalarproduct_int16_c;
2895     c->vector_clip_int32   = vector_clip_int32_c;
2896     c->vector_clipf        = vector_clipf_c;
2897
2898     c->shrink[0] = av_image_copy_plane;
2899     c->shrink[1] = ff_shrink22;
2900     c->shrink[2] = ff_shrink44;
2901     c->shrink[3] = ff_shrink88;
2902
2903     c->add_pixels8 = add_pixels8_c;
2904
2905 #undef FUNC
2906 #undef FUNCC
2907 #define FUNC(f,  depth) f ## _ ## depth
2908 #define FUNCC(f, depth) f ## _ ## depth ## _c
2909
2910     c->draw_edges = FUNCC(draw_edges, 8);
2911
2912     c->clear_block  = FUNCC(clear_block, 8);
2913     c->clear_blocks = FUNCC(clear_blocks, 8);
2914
2915 #define BIT_DEPTH_FUNCS(depth)                  \
2916     c->get_pixels = FUNCC(get_pixels, depth);
2917
2918     switch (avctx->bits_per_raw_sample) {
2919     case 9:
2920     case 10:
2921         BIT_DEPTH_FUNCS(16);
2922         break;
2923     default:
2924         BIT_DEPTH_FUNCS(8);
2925         break;
2926     }
2927
2928     if (ARCH_ARM)
2929         ff_dsputil_init_arm(c, avctx, high_bit_depth);
2930     if (ARCH_BFIN)
2931         ff_dsputil_init_bfin(c, avctx, high_bit_depth);
2932     if (ARCH_PPC)
2933         ff_dsputil_init_ppc(c, avctx, high_bit_depth);
2934     if (ARCH_X86)
2935         ff_dsputil_init_x86(c, avctx, high_bit_depth);
2936
2937     ff_init_scantable_permutation(c->idct_permutation,
2938                                   c->idct_permutation_type);
2939 }