dsputil: Split off HuffYUV decoding bits into their own context
[ffmpeg.git] / libavcodec / dsputil.c
1 /*
2  * DSP utils
3  * Copyright (c) 2000, 2001 Fabrice Bellard
4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * gmc & q-pel & 32/64 bit based MC by Michael Niedermayer <michaelni@gmx.at>
7  *
8  * This file is part of Libav.
9  *
10  * Libav is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * Libav is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with Libav; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 /**
26  * @file
27  * DSP utils
28  */
29
30 #include "libavutil/attributes.h"
31 #include "libavutil/imgutils.h"
32 #include "avcodec.h"
33 #include "copy_block.h"
34 #include "dct.h"
35 #include "dsputil.h"
36 #include "simple_idct.h"
37 #include "faandct.h"
38 #include "faanidct.h"
39 #include "imgconvert.h"
40 #include "mathops.h"
41 #include "mpegvideo.h"
42 #include "config.h"
43
44 uint32_t ff_square_tab[512] = { 0, };
45
46 #define BIT_DEPTH 16
47 #include "dsputilenc_template.c"
48 #undef BIT_DEPTH
49
50 #define BIT_DEPTH 8
51 #include "hpel_template.c"
52 #include "tpel_template.c"
53 #include "dsputil_template.c"
54 #include "dsputilenc_template.c"
55
56 // 0x7f7f7f7f or 0x7f7f7f7f7f7f7f7f or whatever, depending on the cpu's native arithmetic size
57 #define pb_7f (~0UL / 255 * 0x7f)
58 #define pb_80 (~0UL / 255 * 0x80)
59
60 const uint8_t ff_alternate_horizontal_scan[64] = {
61      0,  1,  2,  3,  8,  9, 16, 17,
62     10, 11,  4,  5,  6,  7, 15, 14,
63     13, 12, 19, 18, 24, 25, 32, 33,
64     26, 27, 20, 21, 22, 23, 28, 29,
65     30, 31, 34, 35, 40, 41, 48, 49,
66     42, 43, 36, 37, 38, 39, 44, 45,
67     46, 47, 50, 51, 56, 57, 58, 59,
68     52, 53, 54, 55, 60, 61, 62, 63,
69 };
70
71 const uint8_t ff_alternate_vertical_scan[64] = {
72      0,  8, 16, 24,  1,  9,  2, 10,
73     17, 25, 32, 40, 48, 56, 57, 49,
74     41, 33, 26, 18,  3, 11,  4, 12,
75     19, 27, 34, 42, 50, 58, 35, 43,
76     51, 59, 20, 28,  5, 13,  6, 14,
77     21, 29, 36, 44, 52, 60, 37, 45,
78     53, 61, 22, 30,  7, 15, 23, 31,
79     38, 46, 54, 62, 39, 47, 55, 63,
80 };
81
82 /* Input permutation for the simple_idct_mmx */
83 static const uint8_t simple_mmx_permutation[64] = {
84     0x00, 0x08, 0x04, 0x09, 0x01, 0x0C, 0x05, 0x0D,
85     0x10, 0x18, 0x14, 0x19, 0x11, 0x1C, 0x15, 0x1D,
86     0x20, 0x28, 0x24, 0x29, 0x21, 0x2C, 0x25, 0x2D,
87     0x12, 0x1A, 0x16, 0x1B, 0x13, 0x1E, 0x17, 0x1F,
88     0x02, 0x0A, 0x06, 0x0B, 0x03, 0x0E, 0x07, 0x0F,
89     0x30, 0x38, 0x34, 0x39, 0x31, 0x3C, 0x35, 0x3D,
90     0x22, 0x2A, 0x26, 0x2B, 0x23, 0x2E, 0x27, 0x2F,
91     0x32, 0x3A, 0x36, 0x3B, 0x33, 0x3E, 0x37, 0x3F,
92 };
93
94 static const uint8_t idct_sse2_row_perm[8] = { 0, 4, 1, 5, 2, 6, 3, 7 };
95
96 av_cold void ff_init_scantable(uint8_t *permutation, ScanTable *st,
97                                const uint8_t *src_scantable)
98 {
99     int i, end;
100
101     st->scantable = src_scantable;
102
103     for (i = 0; i < 64; i++) {
104         int j = src_scantable[i];
105         st->permutated[i] = permutation[j];
106     }
107
108     end = -1;
109     for (i = 0; i < 64; i++) {
110         int j = st->permutated[i];
111         if (j > end)
112             end = j;
113         st->raster_end[i] = end;
114     }
115 }
116
117 av_cold void ff_init_scantable_permutation(uint8_t *idct_permutation,
118                                            int idct_permutation_type)
119 {
120     int i;
121
122     switch (idct_permutation_type) {
123     case FF_NO_IDCT_PERM:
124         for (i = 0; i < 64; i++)
125             idct_permutation[i] = i;
126         break;
127     case FF_LIBMPEG2_IDCT_PERM:
128         for (i = 0; i < 64; i++)
129             idct_permutation[i] = (i & 0x38) | ((i & 6) >> 1) | ((i & 1) << 2);
130         break;
131     case FF_SIMPLE_IDCT_PERM:
132         for (i = 0; i < 64; i++)
133             idct_permutation[i] = simple_mmx_permutation[i];
134         break;
135     case FF_TRANSPOSE_IDCT_PERM:
136         for (i = 0; i < 64; i++)
137             idct_permutation[i] = ((i & 7) << 3) | (i >> 3);
138         break;
139     case FF_PARTTRANS_IDCT_PERM:
140         for (i = 0; i < 64; i++)
141             idct_permutation[i] = (i & 0x24) | ((i & 3) << 3) | ((i >> 3) & 3);
142         break;
143     case FF_SSE2_IDCT_PERM:
144         for (i = 0; i < 64; i++)
145             idct_permutation[i] = (i & 0x38) | idct_sse2_row_perm[i & 7];
146         break;
147     default:
148         av_log(NULL, AV_LOG_ERROR,
149                "Internal error, IDCT permutation not set\n");
150     }
151 }
152
153 static int pix_sum_c(uint8_t *pix, int line_size)
154 {
155     int s = 0, i, j;
156
157     for (i = 0; i < 16; i++) {
158         for (j = 0; j < 16; j += 8) {
159             s   += pix[0];
160             s   += pix[1];
161             s   += pix[2];
162             s   += pix[3];
163             s   += pix[4];
164             s   += pix[5];
165             s   += pix[6];
166             s   += pix[7];
167             pix += 8;
168         }
169         pix += line_size - 16;
170     }
171     return s;
172 }
173
174 static int pix_norm1_c(uint8_t *pix, int line_size)
175 {
176     int s = 0, i, j;
177     uint32_t *sq = ff_square_tab + 256;
178
179     for (i = 0; i < 16; i++) {
180         for (j = 0; j < 16; j += 8) {
181 #if 0
182             s += sq[pix[0]];
183             s += sq[pix[1]];
184             s += sq[pix[2]];
185             s += sq[pix[3]];
186             s += sq[pix[4]];
187             s += sq[pix[5]];
188             s += sq[pix[6]];
189             s += sq[pix[7]];
190 #else
191 #if HAVE_FAST_64BIT
192             register uint64_t x = *(uint64_t *) pix;
193             s += sq[x         & 0xff];
194             s += sq[(x >>  8) & 0xff];
195             s += sq[(x >> 16) & 0xff];
196             s += sq[(x >> 24) & 0xff];
197             s += sq[(x >> 32) & 0xff];
198             s += sq[(x >> 40) & 0xff];
199             s += sq[(x >> 48) & 0xff];
200             s += sq[(x >> 56) & 0xff];
201 #else
202             register uint32_t x = *(uint32_t *) pix;
203             s += sq[x         & 0xff];
204             s += sq[(x >>  8) & 0xff];
205             s += sq[(x >> 16) & 0xff];
206             s += sq[(x >> 24) & 0xff];
207             x  = *(uint32_t *) (pix + 4);
208             s += sq[x         & 0xff];
209             s += sq[(x >>  8) & 0xff];
210             s += sq[(x >> 16) & 0xff];
211             s += sq[(x >> 24) & 0xff];
212 #endif
213 #endif
214             pix += 8;
215         }
216         pix += line_size - 16;
217     }
218     return s;
219 }
220
221 static void bswap_buf(uint32_t *dst, const uint32_t *src, int w)
222 {
223     int i;
224
225     for (i = 0; i + 8 <= w; i += 8) {
226         dst[i + 0] = av_bswap32(src[i + 0]);
227         dst[i + 1] = av_bswap32(src[i + 1]);
228         dst[i + 2] = av_bswap32(src[i + 2]);
229         dst[i + 3] = av_bswap32(src[i + 3]);
230         dst[i + 4] = av_bswap32(src[i + 4]);
231         dst[i + 5] = av_bswap32(src[i + 5]);
232         dst[i + 6] = av_bswap32(src[i + 6]);
233         dst[i + 7] = av_bswap32(src[i + 7]);
234     }
235     for (; i < w; i++)
236         dst[i + 0] = av_bswap32(src[i + 0]);
237 }
238
239 static void bswap16_buf(uint16_t *dst, const uint16_t *src, int len)
240 {
241     while (len--)
242         *dst++ = av_bswap16(*src++);
243 }
244
245 static int sse4_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
246                   int line_size, int h)
247 {
248     int s = 0, i;
249     uint32_t *sq = ff_square_tab + 256;
250
251     for (i = 0; i < h; i++) {
252         s    += sq[pix1[0] - pix2[0]];
253         s    += sq[pix1[1] - pix2[1]];
254         s    += sq[pix1[2] - pix2[2]];
255         s    += sq[pix1[3] - pix2[3]];
256         pix1 += line_size;
257         pix2 += line_size;
258     }
259     return s;
260 }
261
262 static int sse8_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
263                   int line_size, int h)
264 {
265     int s = 0, i;
266     uint32_t *sq = ff_square_tab + 256;
267
268     for (i = 0; i < h; i++) {
269         s    += sq[pix1[0] - pix2[0]];
270         s    += sq[pix1[1] - pix2[1]];
271         s    += sq[pix1[2] - pix2[2]];
272         s    += sq[pix1[3] - pix2[3]];
273         s    += sq[pix1[4] - pix2[4]];
274         s    += sq[pix1[5] - pix2[5]];
275         s    += sq[pix1[6] - pix2[6]];
276         s    += sq[pix1[7] - pix2[7]];
277         pix1 += line_size;
278         pix2 += line_size;
279     }
280     return s;
281 }
282
283 static int sse16_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
284                    int line_size, int h)
285 {
286     int s = 0, i;
287     uint32_t *sq = ff_square_tab + 256;
288
289     for (i = 0; i < h; i++) {
290         s += sq[pix1[0]  - pix2[0]];
291         s += sq[pix1[1]  - pix2[1]];
292         s += sq[pix1[2]  - pix2[2]];
293         s += sq[pix1[3]  - pix2[3]];
294         s += sq[pix1[4]  - pix2[4]];
295         s += sq[pix1[5]  - pix2[5]];
296         s += sq[pix1[6]  - pix2[6]];
297         s += sq[pix1[7]  - pix2[7]];
298         s += sq[pix1[8]  - pix2[8]];
299         s += sq[pix1[9]  - pix2[9]];
300         s += sq[pix1[10] - pix2[10]];
301         s += sq[pix1[11] - pix2[11]];
302         s += sq[pix1[12] - pix2[12]];
303         s += sq[pix1[13] - pix2[13]];
304         s += sq[pix1[14] - pix2[14]];
305         s += sq[pix1[15] - pix2[15]];
306
307         pix1 += line_size;
308         pix2 += line_size;
309     }
310     return s;
311 }
312
313 static void diff_pixels_c(int16_t *restrict block, const uint8_t *s1,
314                           const uint8_t *s2, int stride)
315 {
316     int i;
317
318     /* read the pixels */
319     for (i = 0; i < 8; i++) {
320         block[0] = s1[0] - s2[0];
321         block[1] = s1[1] - s2[1];
322         block[2] = s1[2] - s2[2];
323         block[3] = s1[3] - s2[3];
324         block[4] = s1[4] - s2[4];
325         block[5] = s1[5] - s2[5];
326         block[6] = s1[6] - s2[6];
327         block[7] = s1[7] - s2[7];
328         s1      += stride;
329         s2      += stride;
330         block   += 8;
331     }
332 }
333
334 static void put_pixels_clamped_c(const int16_t *block, uint8_t *restrict pixels,
335                                  int line_size)
336 {
337     int i;
338
339     /* read the pixels */
340     for (i = 0; i < 8; i++) {
341         pixels[0] = av_clip_uint8(block[0]);
342         pixels[1] = av_clip_uint8(block[1]);
343         pixels[2] = av_clip_uint8(block[2]);
344         pixels[3] = av_clip_uint8(block[3]);
345         pixels[4] = av_clip_uint8(block[4]);
346         pixels[5] = av_clip_uint8(block[5]);
347         pixels[6] = av_clip_uint8(block[6]);
348         pixels[7] = av_clip_uint8(block[7]);
349
350         pixels += line_size;
351         block  += 8;
352     }
353 }
354
355 static void put_signed_pixels_clamped_c(const int16_t *block,
356                                         uint8_t *restrict pixels,
357                                         int line_size)
358 {
359     int i, j;
360
361     for (i = 0; i < 8; i++) {
362         for (j = 0; j < 8; j++) {
363             if (*block < -128)
364                 *pixels = 0;
365             else if (*block > 127)
366                 *pixels = 255;
367             else
368                 *pixels = (uint8_t) (*block + 128);
369             block++;
370             pixels++;
371         }
372         pixels += (line_size - 8);
373     }
374 }
375
376 static void add_pixels8_c(uint8_t *restrict pixels, int16_t *block,
377                           int line_size)
378 {
379     int i;
380
381     for (i = 0; i < 8; i++) {
382         pixels[0] += block[0];
383         pixels[1] += block[1];
384         pixels[2] += block[2];
385         pixels[3] += block[3];
386         pixels[4] += block[4];
387         pixels[5] += block[5];
388         pixels[6] += block[6];
389         pixels[7] += block[7];
390         pixels    += line_size;
391         block     += 8;
392     }
393 }
394
395 static void add_pixels_clamped_c(const int16_t *block, uint8_t *restrict pixels,
396                                  int line_size)
397 {
398     int i;
399
400     /* read the pixels */
401     for (i = 0; i < 8; i++) {
402         pixels[0] = av_clip_uint8(pixels[0] + block[0]);
403         pixels[1] = av_clip_uint8(pixels[1] + block[1]);
404         pixels[2] = av_clip_uint8(pixels[2] + block[2]);
405         pixels[3] = av_clip_uint8(pixels[3] + block[3]);
406         pixels[4] = av_clip_uint8(pixels[4] + block[4]);
407         pixels[5] = av_clip_uint8(pixels[5] + block[5]);
408         pixels[6] = av_clip_uint8(pixels[6] + block[6]);
409         pixels[7] = av_clip_uint8(pixels[7] + block[7]);
410         pixels   += line_size;
411         block    += 8;
412     }
413 }
414
415 static int sum_abs_dctelem_c(int16_t *block)
416 {
417     int sum = 0, i;
418
419     for (i = 0; i < 64; i++)
420         sum += FFABS(block[i]);
421     return sum;
422 }
423
424 static void fill_block16_c(uint8_t *block, uint8_t value, int line_size, int h)
425 {
426     int i;
427
428     for (i = 0; i < h; i++) {
429         memset(block, value, 16);
430         block += line_size;
431     }
432 }
433
434 static void fill_block8_c(uint8_t *block, uint8_t value, int line_size, int h)
435 {
436     int i;
437
438     for (i = 0; i < h; i++) {
439         memset(block, value, 8);
440         block += line_size;
441     }
442 }
443
444 #define avg2(a, b) ((a + b + 1) >> 1)
445 #define avg4(a, b, c, d) ((a + b + c + d + 2) >> 2)
446
447 static void gmc1_c(uint8_t *dst, uint8_t *src, int stride, int h,
448                    int x16, int y16, int rounder)
449 {
450     const int A = (16 - x16) * (16 - y16);
451     const int B = (x16)      * (16 - y16);
452     const int C = (16 - x16) * (y16);
453     const int D = (x16)      * (y16);
454     int i;
455
456     for (i = 0; i < h; i++) {
457         dst[0] = (A * src[0] + B * src[1] + C * src[stride + 0] + D * src[stride + 1] + rounder) >> 8;
458         dst[1] = (A * src[1] + B * src[2] + C * src[stride + 1] + D * src[stride + 2] + rounder) >> 8;
459         dst[2] = (A * src[2] + B * src[3] + C * src[stride + 2] + D * src[stride + 3] + rounder) >> 8;
460         dst[3] = (A * src[3] + B * src[4] + C * src[stride + 3] + D * src[stride + 4] + rounder) >> 8;
461         dst[4] = (A * src[4] + B * src[5] + C * src[stride + 4] + D * src[stride + 5] + rounder) >> 8;
462         dst[5] = (A * src[5] + B * src[6] + C * src[stride + 5] + D * src[stride + 6] + rounder) >> 8;
463         dst[6] = (A * src[6] + B * src[7] + C * src[stride + 6] + D * src[stride + 7] + rounder) >> 8;
464         dst[7] = (A * src[7] + B * src[8] + C * src[stride + 7] + D * src[stride + 8] + rounder) >> 8;
465         dst   += stride;
466         src   += stride;
467     }
468 }
469
470 void ff_gmc_c(uint8_t *dst, uint8_t *src, int stride, int h, int ox, int oy,
471               int dxx, int dxy, int dyx, int dyy, int shift, int r,
472               int width, int height)
473 {
474     int y, vx, vy;
475     const int s = 1 << shift;
476
477     width--;
478     height--;
479
480     for (y = 0; y < h; y++) {
481         int x;
482
483         vx = ox;
484         vy = oy;
485         for (x = 0; x < 8; x++) { // FIXME: optimize
486             int index;
487             int src_x  = vx >> 16;
488             int src_y  = vy >> 16;
489             int frac_x = src_x & (s - 1);
490             int frac_y = src_y & (s - 1);
491
492             src_x >>= shift;
493             src_y >>= shift;
494
495             if ((unsigned) src_x < width) {
496                 if ((unsigned) src_y < height) {
497                     index = src_x + src_y * stride;
498                     dst[y * stride + x] =
499                         ((src[index]                        * (s - frac_x) +
500                           src[index + 1]          * frac_x) * (s - frac_y) +
501                          (src[index + stride]               * (s - frac_x) +
502                           src[index + stride + 1] * frac_x) *      frac_y  +
503                          r) >> (shift * 2);
504                 } else {
505                     index = src_x + av_clip(src_y, 0, height) * stride;
506                     dst[y * stride + x] =
507                         ((src[index]               * (s - frac_x) +
508                           src[index + 1] * frac_x) *  s           +
509                          r) >> (shift * 2);
510                 }
511             } else {
512                 if ((unsigned) src_y < height) {
513                     index = av_clip(src_x, 0, width) + src_y * stride;
514                     dst[y * stride + x] =
515                         ((src[index]                    * (s - frac_y) +
516                           src[index + stride] * frac_y) *  s           +
517                          r) >> (shift * 2);
518                 } else {
519                     index = av_clip(src_x, 0, width) +
520                             av_clip(src_y, 0, height) * stride;
521                     dst[y * stride + x] = src[index];
522                 }
523             }
524
525             vx += dxx;
526             vy += dyx;
527         }
528         ox += dxy;
529         oy += dyy;
530     }
531 }
532
533 #define QPEL_MC(r, OPNAME, RND, OP)                                           \
534 static void OPNAME ## mpeg4_qpel8_h_lowpass(uint8_t *dst, uint8_t *src,       \
535                                             int dstStride, int srcStride,     \
536                                             int h)                            \
537 {                                                                             \
538     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;                           \
539     int i;                                                                    \
540                                                                               \
541     for (i = 0; i < h; i++) {                                                 \
542         OP(dst[0], (src[0] + src[1]) * 20 - (src[0] + src[2]) * 6 + (src[1] + src[3]) * 3 - (src[2] + src[4])); \
543         OP(dst[1], (src[1] + src[2]) * 20 - (src[0] + src[3]) * 6 + (src[0] + src[4]) * 3 - (src[1] + src[5])); \
544         OP(dst[2], (src[2] + src[3]) * 20 - (src[1] + src[4]) * 6 + (src[0] + src[5]) * 3 - (src[0] + src[6])); \
545         OP(dst[3], (src[3] + src[4]) * 20 - (src[2] + src[5]) * 6 + (src[1] + src[6]) * 3 - (src[0] + src[7])); \
546         OP(dst[4], (src[4] + src[5]) * 20 - (src[3] + src[6]) * 6 + (src[2] + src[7]) * 3 - (src[1] + src[8])); \
547         OP(dst[5], (src[5] + src[6]) * 20 - (src[4] + src[7]) * 6 + (src[3] + src[8]) * 3 - (src[2] + src[8])); \
548         OP(dst[6], (src[6] + src[7]) * 20 - (src[5] + src[8]) * 6 + (src[4] + src[8]) * 3 - (src[3] + src[7])); \
549         OP(dst[7], (src[7] + src[8]) * 20 - (src[6] + src[8]) * 6 + (src[5] + src[7]) * 3 - (src[4] + src[6])); \
550         dst += dstStride;                                                     \
551         src += srcStride;                                                     \
552     }                                                                         \
553 }                                                                             \
554                                                                               \
555 static void OPNAME ## mpeg4_qpel8_v_lowpass(uint8_t *dst, uint8_t *src,       \
556                                             int dstStride, int srcStride)     \
557 {                                                                             \
558     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;                           \
559     const int w = 8;                                                          \
560     int i;                                                                    \
561                                                                               \
562     for (i = 0; i < w; i++) {                                                 \
563         const int src0 = src[0 * srcStride];                                  \
564         const int src1 = src[1 * srcStride];                                  \
565         const int src2 = src[2 * srcStride];                                  \
566         const int src3 = src[3 * srcStride];                                  \
567         const int src4 = src[4 * srcStride];                                  \
568         const int src5 = src[5 * srcStride];                                  \
569         const int src6 = src[6 * srcStride];                                  \
570         const int src7 = src[7 * srcStride];                                  \
571         const int src8 = src[8 * srcStride];                                  \
572         OP(dst[0 * dstStride], (src0 + src1) * 20 - (src0 + src2) * 6 + (src1 + src3) * 3 - (src2 + src4)); \
573         OP(dst[1 * dstStride], (src1 + src2) * 20 - (src0 + src3) * 6 + (src0 + src4) * 3 - (src1 + src5)); \
574         OP(dst[2 * dstStride], (src2 + src3) * 20 - (src1 + src4) * 6 + (src0 + src5) * 3 - (src0 + src6)); \
575         OP(dst[3 * dstStride], (src3 + src4) * 20 - (src2 + src5) * 6 + (src1 + src6) * 3 - (src0 + src7)); \
576         OP(dst[4 * dstStride], (src4 + src5) * 20 - (src3 + src6) * 6 + (src2 + src7) * 3 - (src1 + src8)); \
577         OP(dst[5 * dstStride], (src5 + src6) * 20 - (src4 + src7) * 6 + (src3 + src8) * 3 - (src2 + src8)); \
578         OP(dst[6 * dstStride], (src6 + src7) * 20 - (src5 + src8) * 6 + (src4 + src8) * 3 - (src3 + src7)); \
579         OP(dst[7 * dstStride], (src7 + src8) * 20 - (src6 + src8) * 6 + (src5 + src7) * 3 - (src4 + src6)); \
580         dst++;                                                                \
581         src++;                                                                \
582     }                                                                         \
583 }                                                                             \
584                                                                               \
585 static void OPNAME ## mpeg4_qpel16_h_lowpass(uint8_t *dst, uint8_t *src,      \
586                                              int dstStride, int srcStride,    \
587                                              int h)                           \
588 {                                                                             \
589     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;                           \
590     int i;                                                                    \
591                                                                               \
592     for (i = 0; i < h; i++) {                                                 \
593         OP(dst[0],  (src[0]  + src[1])  * 20 - (src[0]  + src[2])  * 6 + (src[1]  + src[3])  * 3 - (src[2]  + src[4]));  \
594         OP(dst[1],  (src[1]  + src[2])  * 20 - (src[0]  + src[3])  * 6 + (src[0]  + src[4])  * 3 - (src[1]  + src[5]));  \
595         OP(dst[2],  (src[2]  + src[3])  * 20 - (src[1]  + src[4])  * 6 + (src[0]  + src[5])  * 3 - (src[0]  + src[6]));  \
596         OP(dst[3],  (src[3]  + src[4])  * 20 - (src[2]  + src[5])  * 6 + (src[1]  + src[6])  * 3 - (src[0]  + src[7]));  \
597         OP(dst[4],  (src[4]  + src[5])  * 20 - (src[3]  + src[6])  * 6 + (src[2]  + src[7])  * 3 - (src[1]  + src[8]));  \
598         OP(dst[5],  (src[5]  + src[6])  * 20 - (src[4]  + src[7])  * 6 + (src[3]  + src[8])  * 3 - (src[2]  + src[9]));  \
599         OP(dst[6],  (src[6]  + src[7])  * 20 - (src[5]  + src[8])  * 6 + (src[4]  + src[9])  * 3 - (src[3]  + src[10])); \
600         OP(dst[7],  (src[7]  + src[8])  * 20 - (src[6]  + src[9])  * 6 + (src[5]  + src[10]) * 3 - (src[4]  + src[11])); \
601         OP(dst[8],  (src[8]  + src[9])  * 20 - (src[7]  + src[10]) * 6 + (src[6]  + src[11]) * 3 - (src[5]  + src[12])); \
602         OP(dst[9],  (src[9]  + src[10]) * 20 - (src[8]  + src[11]) * 6 + (src[7]  + src[12]) * 3 - (src[6]  + src[13])); \
603         OP(dst[10], (src[10] + src[11]) * 20 - (src[9]  + src[12]) * 6 + (src[8]  + src[13]) * 3 - (src[7]  + src[14])); \
604         OP(dst[11], (src[11] + src[12]) * 20 - (src[10] + src[13]) * 6 + (src[9]  + src[14]) * 3 - (src[8]  + src[15])); \
605         OP(dst[12], (src[12] + src[13]) * 20 - (src[11] + src[14]) * 6 + (src[10] + src[15]) * 3 - (src[9]  + src[16])); \
606         OP(dst[13], (src[13] + src[14]) * 20 - (src[12] + src[15]) * 6 + (src[11] + src[16]) * 3 - (src[10] + src[16])); \
607         OP(dst[14], (src[14] + src[15]) * 20 - (src[13] + src[16]) * 6 + (src[12] + src[16]) * 3 - (src[11] + src[15])); \
608         OP(dst[15], (src[15] + src[16]) * 20 - (src[14] + src[16]) * 6 + (src[13] + src[15]) * 3 - (src[12] + src[14])); \
609         dst += dstStride;                                                     \
610         src += srcStride;                                                     \
611     }                                                                         \
612 }                                                                             \
613                                                                               \
614 static void OPNAME ## mpeg4_qpel16_v_lowpass(uint8_t *dst, uint8_t *src,      \
615                                              int dstStride, int srcStride)    \
616 {                                                                             \
617     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;                           \
618     const int w = 16;                                                         \
619     int i;                                                                    \
620                                                                               \
621     for (i = 0; i < w; i++) {                                                 \
622         const int src0  = src[0  * srcStride];                                \
623         const int src1  = src[1  * srcStride];                                \
624         const int src2  = src[2  * srcStride];                                \
625         const int src3  = src[3  * srcStride];                                \
626         const int src4  = src[4  * srcStride];                                \
627         const int src5  = src[5  * srcStride];                                \
628         const int src6  = src[6  * srcStride];                                \
629         const int src7  = src[7  * srcStride];                                \
630         const int src8  = src[8  * srcStride];                                \
631         const int src9  = src[9  * srcStride];                                \
632         const int src10 = src[10 * srcStride];                                \
633         const int src11 = src[11 * srcStride];                                \
634         const int src12 = src[12 * srcStride];                                \
635         const int src13 = src[13 * srcStride];                                \
636         const int src14 = src[14 * srcStride];                                \
637         const int src15 = src[15 * srcStride];                                \
638         const int src16 = src[16 * srcStride];                                \
639         OP(dst[0  * dstStride], (src0  + src1)  * 20 - (src0  + src2)  * 6 + (src1  + src3)  * 3 - (src2  + src4));  \
640         OP(dst[1  * dstStride], (src1  + src2)  * 20 - (src0  + src3)  * 6 + (src0  + src4)  * 3 - (src1  + src5));  \
641         OP(dst[2  * dstStride], (src2  + src3)  * 20 - (src1  + src4)  * 6 + (src0  + src5)  * 3 - (src0  + src6));  \
642         OP(dst[3  * dstStride], (src3  + src4)  * 20 - (src2  + src5)  * 6 + (src1  + src6)  * 3 - (src0  + src7));  \
643         OP(dst[4  * dstStride], (src4  + src5)  * 20 - (src3  + src6)  * 6 + (src2  + src7)  * 3 - (src1  + src8));  \
644         OP(dst[5  * dstStride], (src5  + src6)  * 20 - (src4  + src7)  * 6 + (src3  + src8)  * 3 - (src2  + src9));  \
645         OP(dst[6  * dstStride], (src6  + src7)  * 20 - (src5  + src8)  * 6 + (src4  + src9)  * 3 - (src3  + src10)); \
646         OP(dst[7  * dstStride], (src7  + src8)  * 20 - (src6  + src9)  * 6 + (src5  + src10) * 3 - (src4  + src11)); \
647         OP(dst[8  * dstStride], (src8  + src9)  * 20 - (src7  + src10) * 6 + (src6  + src11) * 3 - (src5  + src12)); \
648         OP(dst[9  * dstStride], (src9  + src10) * 20 - (src8  + src11) * 6 + (src7  + src12) * 3 - (src6  + src13)); \
649         OP(dst[10 * dstStride], (src10 + src11) * 20 - (src9  + src12) * 6 + (src8  + src13) * 3 - (src7  + src14)); \
650         OP(dst[11 * dstStride], (src11 + src12) * 20 - (src10 + src13) * 6 + (src9  + src14) * 3 - (src8  + src15)); \
651         OP(dst[12 * dstStride], (src12 + src13) * 20 - (src11 + src14) * 6 + (src10 + src15) * 3 - (src9  + src16)); \
652         OP(dst[13 * dstStride], (src13 + src14) * 20 - (src12 + src15) * 6 + (src11 + src16) * 3 - (src10 + src16)); \
653         OP(dst[14 * dstStride], (src14 + src15) * 20 - (src13 + src16) * 6 + (src12 + src16) * 3 - (src11 + src15)); \
654         OP(dst[15 * dstStride], (src15 + src16) * 20 - (src14 + src16) * 6 + (src13 + src15) * 3 - (src12 + src14)); \
655         dst++;                                                                \
656         src++;                                                                \
657     }                                                                         \
658 }                                                                             \
659                                                                               \
660 static void OPNAME ## qpel8_mc10_c(uint8_t *dst, uint8_t *src,                \
661                                    ptrdiff_t stride)                          \
662 {                                                                             \
663     uint8_t half[64];                                                         \
664                                                                               \
665     put ## RND ## mpeg4_qpel8_h_lowpass(half, src, 8, stride, 8);             \
666     OPNAME ## pixels8_l2_8(dst, src, half, stride, stride, 8, 8);             \
667 }                                                                             \
668                                                                               \
669 static void OPNAME ## qpel8_mc20_c(uint8_t *dst, uint8_t *src,                \
670                                    ptrdiff_t stride)                          \
671 {                                                                             \
672     OPNAME ## mpeg4_qpel8_h_lowpass(dst, src, stride, stride, 8);             \
673 }                                                                             \
674                                                                               \
675 static void OPNAME ## qpel8_mc30_c(uint8_t *dst, uint8_t *src,                \
676                                    ptrdiff_t stride)                          \
677 {                                                                             \
678     uint8_t half[64];                                                         \
679                                                                               \
680     put ## RND ## mpeg4_qpel8_h_lowpass(half, src, 8, stride, 8);             \
681     OPNAME ## pixels8_l2_8(dst, src + 1, half, stride, stride, 8, 8);         \
682 }                                                                             \
683                                                                               \
684 static void OPNAME ## qpel8_mc01_c(uint8_t *dst, uint8_t *src,                \
685                                    ptrdiff_t stride)                          \
686 {                                                                             \
687     uint8_t full[16 * 9];                                                     \
688     uint8_t half[64];                                                         \
689                                                                               \
690     copy_block9(full, src, 16, stride, 9);                                    \
691     put ## RND ## mpeg4_qpel8_v_lowpass(half, full, 8, 16);                   \
692     OPNAME ## pixels8_l2_8(dst, full, half, stride, 16, 8, 8);                \
693 }                                                                             \
694                                                                               \
695 static void OPNAME ## qpel8_mc02_c(uint8_t *dst, uint8_t *src,                \
696                                    ptrdiff_t stride)                          \
697 {                                                                             \
698     uint8_t full[16 * 9];                                                     \
699                                                                               \
700     copy_block9(full, src, 16, stride, 9);                                    \
701     OPNAME ## mpeg4_qpel8_v_lowpass(dst, full, stride, 16);                   \
702 }                                                                             \
703                                                                               \
704 static void OPNAME ## qpel8_mc03_c(uint8_t *dst, uint8_t *src,                \
705                                    ptrdiff_t stride)                          \
706 {                                                                             \
707     uint8_t full[16 * 9];                                                     \
708     uint8_t half[64];                                                         \
709                                                                               \
710     copy_block9(full, src, 16, stride, 9);                                    \
711     put ## RND ## mpeg4_qpel8_v_lowpass(half, full, 8, 16);                   \
712     OPNAME ## pixels8_l2_8(dst, full + 16, half, stride, 16, 8, 8);           \
713 }                                                                             \
714                                                                               \
715 void ff_ ## OPNAME ## qpel8_mc11_old_c(uint8_t *dst, uint8_t *src,            \
716                                        ptrdiff_t stride)                      \
717 {                                                                             \
718     uint8_t full[16 * 9];                                                     \
719     uint8_t halfH[72];                                                        \
720     uint8_t halfV[64];                                                        \
721     uint8_t halfHV[64];                                                       \
722                                                                               \
723     copy_block9(full, src, 16, stride, 9);                                    \
724     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
725     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full, 8, 16);                  \
726     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
727     OPNAME ## pixels8_l4_8(dst, full, halfH, halfV, halfHV,                   \
728                            stride, 16, 8, 8, 8, 8);                           \
729 }                                                                             \
730                                                                               \
731 static void OPNAME ## qpel8_mc11_c(uint8_t *dst, uint8_t *src,                \
732                                    ptrdiff_t stride)                          \
733 {                                                                             \
734     uint8_t full[16 * 9];                                                     \
735     uint8_t halfH[72];                                                        \
736     uint8_t halfHV[64];                                                       \
737                                                                               \
738     copy_block9(full, src, 16, stride, 9);                                    \
739     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
740     put ## RND ## pixels8_l2_8(halfH, halfH, full, 8, 8, 16, 9);              \
741     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
742     OPNAME ## pixels8_l2_8(dst, halfH, halfHV, stride, 8, 8, 8);              \
743 }                                                                             \
744                                                                               \
745 void ff_ ## OPNAME ## qpel8_mc31_old_c(uint8_t *dst, uint8_t *src,            \
746                                        ptrdiff_t stride)                      \
747 {                                                                             \
748     uint8_t full[16 * 9];                                                     \
749     uint8_t halfH[72];                                                        \
750     uint8_t halfV[64];                                                        \
751     uint8_t halfHV[64];                                                       \
752                                                                               \
753     copy_block9(full, src, 16, stride, 9);                                    \
754     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
755     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full + 1, 8, 16);              \
756     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
757     OPNAME ## pixels8_l4_8(dst, full + 1, halfH, halfV, halfHV,               \
758                            stride, 16, 8, 8, 8, 8);                           \
759 }                                                                             \
760                                                                               \
761 static void OPNAME ## qpel8_mc31_c(uint8_t *dst, uint8_t *src,                \
762                                    ptrdiff_t stride)                          \
763 {                                                                             \
764     uint8_t full[16 * 9];                                                     \
765     uint8_t halfH[72];                                                        \
766     uint8_t halfHV[64];                                                       \
767                                                                               \
768     copy_block9(full, src, 16, stride, 9);                                    \
769     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
770     put ## RND ## pixels8_l2_8(halfH, halfH, full + 1, 8, 8, 16, 9);          \
771     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
772     OPNAME ## pixels8_l2_8(dst, halfH, halfHV, stride, 8, 8, 8);              \
773 }                                                                             \
774                                                                               \
775 void ff_ ## OPNAME ## qpel8_mc13_old_c(uint8_t *dst, uint8_t *src,            \
776                                        ptrdiff_t stride)                      \
777 {                                                                             \
778     uint8_t full[16 * 9];                                                     \
779     uint8_t halfH[72];                                                        \
780     uint8_t halfV[64];                                                        \
781     uint8_t halfHV[64];                                                       \
782                                                                               \
783     copy_block9(full, src, 16, stride, 9);                                    \
784     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
785     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full, 8, 16);                  \
786     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
787     OPNAME ## pixels8_l4_8(dst, full + 16, halfH + 8, halfV, halfHV,          \
788                            stride, 16, 8, 8, 8, 8);                           \
789 }                                                                             \
790                                                                               \
791 static void OPNAME ## qpel8_mc13_c(uint8_t *dst, uint8_t *src,                \
792                                    ptrdiff_t stride)                          \
793 {                                                                             \
794     uint8_t full[16 * 9];                                                     \
795     uint8_t halfH[72];                                                        \
796     uint8_t halfHV[64];                                                       \
797                                                                               \
798     copy_block9(full, src, 16, stride, 9);                                    \
799     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
800     put ## RND ## pixels8_l2_8(halfH, halfH, full, 8, 8, 16, 9);              \
801     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
802     OPNAME ## pixels8_l2_8(dst, halfH + 8, halfHV, stride, 8, 8, 8);          \
803 }                                                                             \
804                                                                               \
805 void ff_ ## OPNAME ## qpel8_mc33_old_c(uint8_t *dst, uint8_t *src,            \
806                                        ptrdiff_t stride)                      \
807 {                                                                             \
808     uint8_t full[16 * 9];                                                     \
809     uint8_t halfH[72];                                                        \
810     uint8_t halfV[64];                                                        \
811     uint8_t halfHV[64];                                                       \
812                                                                               \
813     copy_block9(full, src, 16, stride, 9);                                    \
814     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
815     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full + 1, 8, 16);              \
816     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
817     OPNAME ## pixels8_l4_8(dst, full + 17, halfH + 8, halfV, halfHV,          \
818                            stride, 16, 8, 8, 8, 8);                           \
819 }                                                                             \
820                                                                               \
821 static void OPNAME ## qpel8_mc33_c(uint8_t *dst, uint8_t *src,                \
822                                    ptrdiff_t stride)                          \
823 {                                                                             \
824     uint8_t full[16 * 9];                                                     \
825     uint8_t halfH[72];                                                        \
826     uint8_t halfHV[64];                                                       \
827                                                                               \
828     copy_block9(full, src, 16, stride, 9);                                    \
829     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
830     put ## RND ## pixels8_l2_8(halfH, halfH, full + 1, 8, 8, 16, 9);          \
831     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
832     OPNAME ## pixels8_l2_8(dst, halfH + 8, halfHV, stride, 8, 8, 8);          \
833 }                                                                             \
834                                                                               \
835 static void OPNAME ## qpel8_mc21_c(uint8_t *dst, uint8_t *src,                \
836                                    ptrdiff_t stride)                          \
837 {                                                                             \
838     uint8_t halfH[72];                                                        \
839     uint8_t halfHV[64];                                                       \
840                                                                               \
841     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, src, 8, stride, 9);            \
842     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
843     OPNAME ## pixels8_l2_8(dst, halfH, halfHV, stride, 8, 8, 8);              \
844 }                                                                             \
845                                                                               \
846 static void OPNAME ## qpel8_mc23_c(uint8_t *dst, uint8_t *src,                \
847                                    ptrdiff_t stride)                          \
848 {                                                                             \
849     uint8_t halfH[72];                                                        \
850     uint8_t halfHV[64];                                                       \
851                                                                               \
852     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, src, 8, stride, 9);            \
853     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
854     OPNAME ## pixels8_l2_8(dst, halfH + 8, halfHV, stride, 8, 8, 8);          \
855 }                                                                             \
856                                                                               \
857 void ff_ ## OPNAME ## qpel8_mc12_old_c(uint8_t *dst, uint8_t *src,            \
858                                        ptrdiff_t stride)                      \
859 {                                                                             \
860     uint8_t full[16 * 9];                                                     \
861     uint8_t halfH[72];                                                        \
862     uint8_t halfV[64];                                                        \
863     uint8_t halfHV[64];                                                       \
864                                                                               \
865     copy_block9(full, src, 16, stride, 9);                                    \
866     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
867     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full, 8, 16);                  \
868     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
869     OPNAME ## pixels8_l2_8(dst, halfV, halfHV, stride, 8, 8, 8);              \
870 }                                                                             \
871                                                                               \
872 static void OPNAME ## qpel8_mc12_c(uint8_t *dst, uint8_t *src,                \
873                                    ptrdiff_t stride)                          \
874 {                                                                             \
875     uint8_t full[16 * 9];                                                     \
876     uint8_t halfH[72];                                                        \
877                                                                               \
878     copy_block9(full, src, 16, stride, 9);                                    \
879     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
880     put ## RND ## pixels8_l2_8(halfH, halfH, full, 8, 8, 16, 9);              \
881     OPNAME ## mpeg4_qpel8_v_lowpass(dst, halfH, stride, 8);                   \
882 }                                                                             \
883                                                                               \
884 void ff_ ## OPNAME ## qpel8_mc32_old_c(uint8_t *dst, uint8_t *src,            \
885                                        ptrdiff_t stride)                      \
886 {                                                                             \
887     uint8_t full[16 * 9];                                                     \
888     uint8_t halfH[72];                                                        \
889     uint8_t halfV[64];                                                        \
890     uint8_t halfHV[64];                                                       \
891                                                                               \
892     copy_block9(full, src, 16, stride, 9);                                    \
893     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
894     put ## RND ## mpeg4_qpel8_v_lowpass(halfV, full + 1, 8, 16);              \
895     put ## RND ## mpeg4_qpel8_v_lowpass(halfHV, halfH, 8, 8);                 \
896     OPNAME ## pixels8_l2_8(dst, halfV, halfHV, stride, 8, 8, 8);              \
897 }                                                                             \
898                                                                               \
899 static void OPNAME ## qpel8_mc32_c(uint8_t *dst, uint8_t *src,                \
900                                    ptrdiff_t stride)                          \
901 {                                                                             \
902     uint8_t full[16 * 9];                                                     \
903     uint8_t halfH[72];                                                        \
904                                                                               \
905     copy_block9(full, src, 16, stride, 9);                                    \
906     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, full, 8, 16, 9);               \
907     put ## RND ## pixels8_l2_8(halfH, halfH, full + 1, 8, 8, 16, 9);          \
908     OPNAME ## mpeg4_qpel8_v_lowpass(dst, halfH, stride, 8);                   \
909 }                                                                             \
910                                                                               \
911 static void OPNAME ## qpel8_mc22_c(uint8_t *dst, uint8_t *src,                \
912                                    ptrdiff_t stride)                          \
913 {                                                                             \
914     uint8_t halfH[72];                                                        \
915                                                                               \
916     put ## RND ## mpeg4_qpel8_h_lowpass(halfH, src, 8, stride, 9);            \
917     OPNAME ## mpeg4_qpel8_v_lowpass(dst, halfH, stride, 8);                   \
918 }                                                                             \
919                                                                               \
920 static void OPNAME ## qpel16_mc10_c(uint8_t *dst, uint8_t *src,               \
921                                     ptrdiff_t stride)                         \
922 {                                                                             \
923     uint8_t half[256];                                                        \
924                                                                               \
925     put ## RND ## mpeg4_qpel16_h_lowpass(half, src, 16, stride, 16);          \
926     OPNAME ## pixels16_l2_8(dst, src, half, stride, stride, 16, 16);          \
927 }                                                                             \
928                                                                               \
929 static void OPNAME ## qpel16_mc20_c(uint8_t *dst, uint8_t *src,               \
930                                     ptrdiff_t stride)                         \
931 {                                                                             \
932     OPNAME ## mpeg4_qpel16_h_lowpass(dst, src, stride, stride, 16);           \
933 }                                                                             \
934                                                                               \
935 static void OPNAME ## qpel16_mc30_c(uint8_t *dst, uint8_t *src,               \
936                                     ptrdiff_t stride)                         \
937 {                                                                             \
938     uint8_t half[256];                                                        \
939                                                                               \
940     put ## RND ## mpeg4_qpel16_h_lowpass(half, src, 16, stride, 16);          \
941     OPNAME ## pixels16_l2_8(dst, src + 1, half, stride, stride, 16, 16);      \
942 }                                                                             \
943                                                                               \
944 static void OPNAME ## qpel16_mc01_c(uint8_t *dst, uint8_t *src,               \
945                                     ptrdiff_t stride)                         \
946 {                                                                             \
947     uint8_t full[24 * 17];                                                    \
948     uint8_t half[256];                                                        \
949                                                                               \
950     copy_block17(full, src, 24, stride, 17);                                  \
951     put ## RND ## mpeg4_qpel16_v_lowpass(half, full, 16, 24);                 \
952     OPNAME ## pixels16_l2_8(dst, full, half, stride, 24, 16, 16);             \
953 }                                                                             \
954                                                                               \
955 static void OPNAME ## qpel16_mc02_c(uint8_t *dst, uint8_t *src,               \
956                                     ptrdiff_t stride)                         \
957 {                                                                             \
958     uint8_t full[24 * 17];                                                    \
959                                                                               \
960     copy_block17(full, src, 24, stride, 17);                                  \
961     OPNAME ## mpeg4_qpel16_v_lowpass(dst, full, stride, 24);                  \
962 }                                                                             \
963                                                                               \
964 static void OPNAME ## qpel16_mc03_c(uint8_t *dst, uint8_t *src,               \
965                                     ptrdiff_t stride)                         \
966 {                                                                             \
967     uint8_t full[24 * 17];                                                    \
968     uint8_t half[256];                                                        \
969                                                                               \
970     copy_block17(full, src, 24, stride, 17);                                  \
971     put ## RND ## mpeg4_qpel16_v_lowpass(half, full, 16, 24);                 \
972     OPNAME ## pixels16_l2_8(dst, full + 24, half, stride, 24, 16, 16);        \
973 }                                                                             \
974                                                                               \
975 void ff_ ## OPNAME ## qpel16_mc11_old_c(uint8_t *dst, uint8_t *src,           \
976                                         ptrdiff_t stride)                     \
977 {                                                                             \
978     uint8_t full[24 * 17];                                                    \
979     uint8_t halfH[272];                                                       \
980     uint8_t halfV[256];                                                       \
981     uint8_t halfHV[256];                                                      \
982                                                                               \
983     copy_block17(full, src, 24, stride, 17);                                  \
984     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
985     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full, 16, 24);                \
986     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
987     OPNAME ## pixels16_l4_8(dst, full, halfH, halfV, halfHV,                  \
988                             stride, 24, 16, 16, 16, 16);                      \
989 }                                                                             \
990                                                                               \
991 static void OPNAME ## qpel16_mc11_c(uint8_t *dst, uint8_t *src,               \
992                                     ptrdiff_t stride)                         \
993 {                                                                             \
994     uint8_t full[24 * 17];                                                    \
995     uint8_t halfH[272];                                                       \
996     uint8_t halfHV[256];                                                      \
997                                                                               \
998     copy_block17(full, src, 24, stride, 17);                                  \
999     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1000     put ## RND ## pixels16_l2_8(halfH, halfH, full, 16, 16, 24, 17);          \
1001     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1002     OPNAME ## pixels16_l2_8(dst, halfH, halfHV, stride, 16, 16, 16);          \
1003 }                                                                             \
1004                                                                               \
1005 void ff_ ## OPNAME ## qpel16_mc31_old_c(uint8_t *dst, uint8_t *src,           \
1006                                         ptrdiff_t stride)                     \
1007 {                                                                             \
1008     uint8_t full[24 * 17];                                                    \
1009     uint8_t halfH[272];                                                       \
1010     uint8_t halfV[256];                                                       \
1011     uint8_t halfHV[256];                                                      \
1012                                                                               \
1013     copy_block17(full, src, 24, stride, 17);                                  \
1014     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1015     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full + 1, 16, 24);            \
1016     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1017     OPNAME ## pixels16_l4_8(dst, full + 1, halfH, halfV, halfHV,              \
1018                             stride, 24, 16, 16, 16, 16);                      \
1019 }                                                                             \
1020                                                                               \
1021 static void OPNAME ## qpel16_mc31_c(uint8_t *dst, uint8_t *src,               \
1022                                     ptrdiff_t stride)                         \
1023 {                                                                             \
1024     uint8_t full[24 * 17];                                                    \
1025     uint8_t halfH[272];                                                       \
1026     uint8_t halfHV[256];                                                      \
1027                                                                               \
1028     copy_block17(full, src, 24, stride, 17);                                  \
1029     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1030     put ## RND ## pixels16_l2_8(halfH, halfH, full + 1, 16, 16, 24, 17);      \
1031     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1032     OPNAME ## pixels16_l2_8(dst, halfH, halfHV, stride, 16, 16, 16);          \
1033 }                                                                             \
1034                                                                               \
1035 void ff_ ## OPNAME ## qpel16_mc13_old_c(uint8_t *dst, uint8_t *src,           \
1036                                         ptrdiff_t stride)                     \
1037 {                                                                             \
1038     uint8_t full[24 * 17];                                                    \
1039     uint8_t halfH[272];                                                       \
1040     uint8_t halfV[256];                                                       \
1041     uint8_t halfHV[256];                                                      \
1042                                                                               \
1043     copy_block17(full, src, 24, stride, 17);                                  \
1044     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1045     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full, 16, 24);                \
1046     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1047     OPNAME ## pixels16_l4_8(dst, full + 24, halfH + 16, halfV, halfHV,        \
1048                             stride, 24, 16, 16, 16, 16);                      \
1049 }                                                                             \
1050                                                                               \
1051 static void OPNAME ## qpel16_mc13_c(uint8_t *dst, uint8_t *src,               \
1052                                     ptrdiff_t stride)                         \
1053 {                                                                             \
1054     uint8_t full[24 * 17];                                                    \
1055     uint8_t halfH[272];                                                       \
1056     uint8_t halfHV[256];                                                      \
1057                                                                               \
1058     copy_block17(full, src, 24, stride, 17);                                  \
1059     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1060     put ## RND ## pixels16_l2_8(halfH, halfH, full, 16, 16, 24, 17);          \
1061     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1062     OPNAME ## pixels16_l2_8(dst, halfH + 16, halfHV, stride, 16, 16, 16);     \
1063 }                                                                             \
1064                                                                               \
1065 void ff_ ## OPNAME ## qpel16_mc33_old_c(uint8_t *dst, uint8_t *src,           \
1066                                         ptrdiff_t stride)                     \
1067 {                                                                             \
1068     uint8_t full[24 * 17];                                                    \
1069     uint8_t halfH[272];                                                       \
1070     uint8_t halfV[256];                                                       \
1071     uint8_t halfHV[256];                                                      \
1072                                                                               \
1073     copy_block17(full, src, 24, stride, 17);                                  \
1074     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1075     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full + 1, 16, 24);            \
1076     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1077     OPNAME ## pixels16_l4_8(dst, full + 25, halfH + 16, halfV, halfHV,        \
1078                             stride, 24, 16, 16, 16, 16);                      \
1079 }                                                                             \
1080                                                                               \
1081 static void OPNAME ## qpel16_mc33_c(uint8_t *dst, uint8_t *src,               \
1082                                     ptrdiff_t stride)                         \
1083 {                                                                             \
1084     uint8_t full[24 * 17];                                                    \
1085     uint8_t halfH[272];                                                       \
1086     uint8_t halfHV[256];                                                      \
1087                                                                               \
1088     copy_block17(full, src, 24, stride, 17);                                  \
1089     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1090     put ## RND ## pixels16_l2_8(halfH, halfH, full + 1, 16, 16, 24, 17);      \
1091     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1092     OPNAME ## pixels16_l2_8(dst, halfH + 16, halfHV, stride, 16, 16, 16);     \
1093 }                                                                             \
1094                                                                               \
1095 static void OPNAME ## qpel16_mc21_c(uint8_t *dst, uint8_t *src,               \
1096                                     ptrdiff_t stride)                         \
1097 {                                                                             \
1098     uint8_t halfH[272];                                                       \
1099     uint8_t halfHV[256];                                                      \
1100                                                                               \
1101     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, src, 16, stride, 17);         \
1102     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1103     OPNAME ## pixels16_l2_8(dst, halfH, halfHV, stride, 16, 16, 16);          \
1104 }                                                                             \
1105                                                                               \
1106 static void OPNAME ## qpel16_mc23_c(uint8_t *dst, uint8_t *src,               \
1107                                     ptrdiff_t stride)                         \
1108 {                                                                             \
1109     uint8_t halfH[272];                                                       \
1110     uint8_t halfHV[256];                                                      \
1111                                                                               \
1112     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, src, 16, stride, 17);         \
1113     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1114     OPNAME ## pixels16_l2_8(dst, halfH + 16, halfHV, stride, 16, 16, 16);     \
1115 }                                                                             \
1116                                                                               \
1117 void ff_ ## OPNAME ## qpel16_mc12_old_c(uint8_t *dst, uint8_t *src,           \
1118                                         ptrdiff_t stride)                     \
1119 {                                                                             \
1120     uint8_t full[24 * 17];                                                    \
1121     uint8_t halfH[272];                                                       \
1122     uint8_t halfV[256];                                                       \
1123     uint8_t halfHV[256];                                                      \
1124                                                                               \
1125     copy_block17(full, src, 24, stride, 17);                                  \
1126     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1127     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full, 16, 24);                \
1128     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1129     OPNAME ## pixels16_l2_8(dst, halfV, halfHV, stride, 16, 16, 16);          \
1130 }                                                                             \
1131                                                                               \
1132 static void OPNAME ## qpel16_mc12_c(uint8_t *dst, uint8_t *src,               \
1133                                     ptrdiff_t stride)                         \
1134 {                                                                             \
1135     uint8_t full[24 * 17];                                                    \
1136     uint8_t halfH[272];                                                       \
1137                                                                               \
1138     copy_block17(full, src, 24, stride, 17);                                  \
1139     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1140     put ## RND ## pixels16_l2_8(halfH, halfH, full, 16, 16, 24, 17);          \
1141     OPNAME ## mpeg4_qpel16_v_lowpass(dst, halfH, stride, 16);                 \
1142 }                                                                             \
1143                                                                               \
1144 void ff_ ## OPNAME ## qpel16_mc32_old_c(uint8_t *dst, uint8_t *src,           \
1145                                         ptrdiff_t stride)                     \
1146 {                                                                             \
1147     uint8_t full[24 * 17];                                                    \
1148     uint8_t halfH[272];                                                       \
1149     uint8_t halfV[256];                                                       \
1150     uint8_t halfHV[256];                                                      \
1151                                                                               \
1152     copy_block17(full, src, 24, stride, 17);                                  \
1153     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1154     put ## RND ## mpeg4_qpel16_v_lowpass(halfV, full + 1, 16, 24);            \
1155     put ## RND ## mpeg4_qpel16_v_lowpass(halfHV, halfH, 16, 16);              \
1156     OPNAME ## pixels16_l2_8(dst, halfV, halfHV, stride, 16, 16, 16);          \
1157 }                                                                             \
1158                                                                               \
1159 static void OPNAME ## qpel16_mc32_c(uint8_t *dst, uint8_t *src,               \
1160                                     ptrdiff_t stride)                         \
1161 {                                                                             \
1162     uint8_t full[24 * 17];                                                    \
1163     uint8_t halfH[272];                                                       \
1164                                                                               \
1165     copy_block17(full, src, 24, stride, 17);                                  \
1166     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, full, 16, 24, 17);            \
1167     put ## RND ## pixels16_l2_8(halfH, halfH, full + 1, 16, 16, 24, 17);      \
1168     OPNAME ## mpeg4_qpel16_v_lowpass(dst, halfH, stride, 16);                 \
1169 }                                                                             \
1170                                                                               \
1171 static void OPNAME ## qpel16_mc22_c(uint8_t *dst, uint8_t *src,               \
1172                                     ptrdiff_t stride)                         \
1173 {                                                                             \
1174     uint8_t halfH[272];                                                       \
1175                                                                               \
1176     put ## RND ## mpeg4_qpel16_h_lowpass(halfH, src, 16, stride, 17);         \
1177     OPNAME ## mpeg4_qpel16_v_lowpass(dst, halfH, stride, 16);                 \
1178 }
1179
1180 #define op_avg(a, b)        a = (((a) + cm[((b) + 16) >> 5] + 1) >> 1)
1181 #define op_avg_no_rnd(a, b) a = (((a) + cm[((b) + 15) >> 5])     >> 1)
1182 #define op_put(a, b)        a = cm[((b) + 16) >> 5]
1183 #define op_put_no_rnd(a, b) a = cm[((b) + 15) >> 5]
1184
1185 QPEL_MC(0, put_, _, op_put)
1186 QPEL_MC(1, put_no_rnd_, _no_rnd_, op_put_no_rnd)
1187 QPEL_MC(0, avg_, _, op_avg)
1188
1189 #undef op_avg
1190 #undef op_put
1191 #undef op_put_no_rnd
1192
1193 void ff_put_pixels8x8_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1194 {
1195     put_pixels8_8_c(dst, src, stride, 8);
1196 }
1197
1198 void ff_avg_pixels8x8_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1199 {
1200     avg_pixels8_8_c(dst, src, stride, 8);
1201 }
1202
1203 void ff_put_pixels16x16_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1204 {
1205     put_pixels16_8_c(dst, src, stride, 16);
1206 }
1207
1208 void ff_avg_pixels16x16_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1209 {
1210     avg_pixels16_8_c(dst, src, stride, 16);
1211 }
1212
1213 #define put_qpel8_mc00_c         ff_put_pixels8x8_c
1214 #define avg_qpel8_mc00_c         ff_avg_pixels8x8_c
1215 #define put_qpel16_mc00_c        ff_put_pixels16x16_c
1216 #define avg_qpel16_mc00_c        ff_avg_pixels16x16_c
1217 #define put_no_rnd_qpel8_mc00_c  ff_put_pixels8x8_c
1218 #define put_no_rnd_qpel16_mc00_c ff_put_pixels16x16_c
1219
1220 static void wmv2_mspel8_h_lowpass(uint8_t *dst, uint8_t *src,
1221                                   int dstStride, int srcStride, int h)
1222 {
1223     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;
1224     int i;
1225
1226     for (i = 0; i < h; i++) {
1227         dst[0] = cm[(9 * (src[0] + src[1]) - (src[-1] + src[2]) + 8) >> 4];
1228         dst[1] = cm[(9 * (src[1] + src[2]) - (src[0]  + src[3]) + 8) >> 4];
1229         dst[2] = cm[(9 * (src[2] + src[3]) - (src[1]  + src[4]) + 8) >> 4];
1230         dst[3] = cm[(9 * (src[3] + src[4]) - (src[2]  + src[5]) + 8) >> 4];
1231         dst[4] = cm[(9 * (src[4] + src[5]) - (src[3]  + src[6]) + 8) >> 4];
1232         dst[5] = cm[(9 * (src[5] + src[6]) - (src[4]  + src[7]) + 8) >> 4];
1233         dst[6] = cm[(9 * (src[6] + src[7]) - (src[5]  + src[8]) + 8) >> 4];
1234         dst[7] = cm[(9 * (src[7] + src[8]) - (src[6]  + src[9]) + 8) >> 4];
1235         dst   += dstStride;
1236         src   += srcStride;
1237     }
1238 }
1239
1240 static void wmv2_mspel8_v_lowpass(uint8_t *dst, uint8_t *src,
1241                                   int dstStride, int srcStride, int w)
1242 {
1243     const uint8_t *cm = ff_crop_tab + MAX_NEG_CROP;
1244     int i;
1245
1246     for (i = 0; i < w; i++) {
1247         const int src_1 = src[-srcStride];
1248         const int src0  = src[0];
1249         const int src1  = src[srcStride];
1250         const int src2  = src[2 * srcStride];
1251         const int src3  = src[3 * srcStride];
1252         const int src4  = src[4 * srcStride];
1253         const int src5  = src[5 * srcStride];
1254         const int src6  = src[6 * srcStride];
1255         const int src7  = src[7 * srcStride];
1256         const int src8  = src[8 * srcStride];
1257         const int src9  = src[9 * srcStride];
1258         dst[0 * dstStride] = cm[(9 * (src0 + src1) - (src_1 + src2) + 8) >> 4];
1259         dst[1 * dstStride] = cm[(9 * (src1 + src2) - (src0  + src3) + 8) >> 4];
1260         dst[2 * dstStride] = cm[(9 * (src2 + src3) - (src1  + src4) + 8) >> 4];
1261         dst[3 * dstStride] = cm[(9 * (src3 + src4) - (src2  + src5) + 8) >> 4];
1262         dst[4 * dstStride] = cm[(9 * (src4 + src5) - (src3  + src6) + 8) >> 4];
1263         dst[5 * dstStride] = cm[(9 * (src5 + src6) - (src4  + src7) + 8) >> 4];
1264         dst[6 * dstStride] = cm[(9 * (src6 + src7) - (src5  + src8) + 8) >> 4];
1265         dst[7 * dstStride] = cm[(9 * (src7 + src8) - (src6  + src9) + 8) >> 4];
1266         src++;
1267         dst++;
1268     }
1269 }
1270
1271 static void put_mspel8_mc10_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1272 {
1273     uint8_t half[64];
1274
1275     wmv2_mspel8_h_lowpass(half, src, 8, stride, 8);
1276     put_pixels8_l2_8(dst, src, half, stride, stride, 8, 8);
1277 }
1278
1279 static void put_mspel8_mc20_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1280 {
1281     wmv2_mspel8_h_lowpass(dst, src, stride, stride, 8);
1282 }
1283
1284 static void put_mspel8_mc30_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1285 {
1286     uint8_t half[64];
1287
1288     wmv2_mspel8_h_lowpass(half, src, 8, stride, 8);
1289     put_pixels8_l2_8(dst, src + 1, half, stride, stride, 8, 8);
1290 }
1291
1292 static void put_mspel8_mc02_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1293 {
1294     wmv2_mspel8_v_lowpass(dst, src, stride, stride, 8);
1295 }
1296
1297 static void put_mspel8_mc12_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1298 {
1299     uint8_t halfH[88];
1300     uint8_t halfV[64];
1301     uint8_t halfHV[64];
1302
1303     wmv2_mspel8_h_lowpass(halfH, src - stride, 8, stride, 11);
1304     wmv2_mspel8_v_lowpass(halfV, src, 8, stride, 8);
1305     wmv2_mspel8_v_lowpass(halfHV, halfH + 8, 8, 8, 8);
1306     put_pixels8_l2_8(dst, halfV, halfHV, stride, 8, 8, 8);
1307 }
1308
1309 static void put_mspel8_mc32_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1310 {
1311     uint8_t halfH[88];
1312     uint8_t halfV[64];
1313     uint8_t halfHV[64];
1314
1315     wmv2_mspel8_h_lowpass(halfH, src - stride, 8, stride, 11);
1316     wmv2_mspel8_v_lowpass(halfV, src + 1, 8, stride, 8);
1317     wmv2_mspel8_v_lowpass(halfHV, halfH + 8, 8, 8, 8);
1318     put_pixels8_l2_8(dst, halfV, halfHV, stride, 8, 8, 8);
1319 }
1320
1321 static void put_mspel8_mc22_c(uint8_t *dst, uint8_t *src, ptrdiff_t stride)
1322 {
1323     uint8_t halfH[88];
1324
1325     wmv2_mspel8_h_lowpass(halfH, src - stride, 8, stride, 11);
1326     wmv2_mspel8_v_lowpass(dst, halfH + 8, stride, 8, 8);
1327 }
1328
1329 static inline int pix_abs16_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
1330                               int line_size, int h)
1331 {
1332     int s = 0, i;
1333
1334     for (i = 0; i < h; i++) {
1335         s    += abs(pix1[0]  - pix2[0]);
1336         s    += abs(pix1[1]  - pix2[1]);
1337         s    += abs(pix1[2]  - pix2[2]);
1338         s    += abs(pix1[3]  - pix2[3]);
1339         s    += abs(pix1[4]  - pix2[4]);
1340         s    += abs(pix1[5]  - pix2[5]);
1341         s    += abs(pix1[6]  - pix2[6]);
1342         s    += abs(pix1[7]  - pix2[7]);
1343         s    += abs(pix1[8]  - pix2[8]);
1344         s    += abs(pix1[9]  - pix2[9]);
1345         s    += abs(pix1[10] - pix2[10]);
1346         s    += abs(pix1[11] - pix2[11]);
1347         s    += abs(pix1[12] - pix2[12]);
1348         s    += abs(pix1[13] - pix2[13]);
1349         s    += abs(pix1[14] - pix2[14]);
1350         s    += abs(pix1[15] - pix2[15]);
1351         pix1 += line_size;
1352         pix2 += line_size;
1353     }
1354     return s;
1355 }
1356
1357 static int pix_abs16_x2_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
1358                           int line_size, int h)
1359 {
1360     int s = 0, i;
1361
1362     for (i = 0; i < h; i++) {
1363         s    += abs(pix1[0]  - avg2(pix2[0],  pix2[1]));
1364         s    += abs(pix1[1]  - avg2(pix2[1],  pix2[2]));
1365         s    += abs(pix1[2]  - avg2(pix2[2],  pix2[3]));
1366         s    += abs(pix1[3]  - avg2(pix2[3],  pix2[4]));
1367         s    += abs(pix1[4]  - avg2(pix2[4],  pix2[5]));
1368         s    += abs(pix1[5]  - avg2(pix2[5],  pix2[6]));
1369         s    += abs(pix1[6]  - avg2(pix2[6],  pix2[7]));
1370         s    += abs(pix1[7]  - avg2(pix2[7],  pix2[8]));
1371         s    += abs(pix1[8]  - avg2(pix2[8],  pix2[9]));
1372         s    += abs(pix1[9]  - avg2(pix2[9],  pix2[10]));
1373         s    += abs(pix1[10] - avg2(pix2[10], pix2[11]));
1374         s    += abs(pix1[11] - avg2(pix2[11], pix2[12]));
1375         s    += abs(pix1[12] - avg2(pix2[12], pix2[13]));
1376         s    += abs(pix1[13] - avg2(pix2[13], pix2[14]));
1377         s    += abs(pix1[14] - avg2(pix2[14], pix2[15]));
1378         s    += abs(pix1[15] - avg2(pix2[15], pix2[16]));
1379         pix1 += line_size;
1380         pix2 += line_size;
1381     }
1382     return s;
1383 }
1384
1385 static int pix_abs16_y2_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
1386                           int line_size, int h)
1387 {
1388     int s = 0, i;
1389     uint8_t *pix3 = pix2 + line_size;
1390
1391     for (i = 0; i < h; i++) {
1392         s    += abs(pix1[0]  - avg2(pix2[0],  pix3[0]));
1393         s    += abs(pix1[1]  - avg2(pix2[1],  pix3[1]));
1394         s    += abs(pix1[2]  - avg2(pix2[2],  pix3[2]));
1395         s    += abs(pix1[3]  - avg2(pix2[3],  pix3[3]));
1396         s    += abs(pix1[4]  - avg2(pix2[4],  pix3[4]));
1397         s    += abs(pix1[5]  - avg2(pix2[5],  pix3[5]));
1398         s    += abs(pix1[6]  - avg2(pix2[6],  pix3[6]));
1399         s    += abs(pix1[7]  - avg2(pix2[7],  pix3[7]));
1400         s    += abs(pix1[8]  - avg2(pix2[8],  pix3[8]));
1401         s    += abs(pix1[9]  - avg2(pix2[9],  pix3[9]));
1402         s    += abs(pix1[10] - avg2(pix2[10], pix3[10]));
1403         s    += abs(pix1[11] - avg2(pix2[11], pix3[11]));
1404         s    += abs(pix1[12] - avg2(pix2[12], pix3[12]));
1405         s    += abs(pix1[13] - avg2(pix2[13], pix3[13]));
1406         s    += abs(pix1[14] - avg2(pix2[14], pix3[14]));
1407         s    += abs(pix1[15] - avg2(pix2[15], pix3[15]));
1408         pix1 += line_size;
1409         pix2 += line_size;
1410         pix3 += line_size;
1411     }
1412     return s;
1413 }
1414
1415 static int pix_abs16_xy2_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
1416                            int line_size, int h)
1417 {
1418     int s = 0, i;
1419     uint8_t *pix3 = pix2 + line_size;
1420
1421     for (i = 0; i < h; i++) {
1422         s    += abs(pix1[0]  - avg4(pix2[0],  pix2[1],  pix3[0],  pix3[1]));
1423         s    += abs(pix1[1]  - avg4(pix2[1],  pix2[2],  pix3[1],  pix3[2]));
1424         s    += abs(pix1[2]  - avg4(pix2[2],  pix2[3],  pix3[2],  pix3[3]));
1425         s    += abs(pix1[3]  - avg4(pix2[3],  pix2[4],  pix3[3],  pix3[4]));
1426         s    += abs(pix1[4]  - avg4(pix2[4],  pix2[5],  pix3[4],  pix3[5]));
1427         s    += abs(pix1[5]  - avg4(pix2[5],  pix2[6],  pix3[5],  pix3[6]));
1428         s    += abs(pix1[6]  - avg4(pix2[6],  pix2[7],  pix3[6],  pix3[7]));
1429         s    += abs(pix1[7]  - avg4(pix2[7],  pix2[8],  pix3[7],  pix3[8]));
1430         s    += abs(pix1[8]  - avg4(pix2[8],  pix2[9],  pix3[8],  pix3[9]));
1431         s    += abs(pix1[9]  - avg4(pix2[9],  pix2[10], pix3[9],  pix3[10]));
1432         s    += abs(pix1[10] - avg4(pix2[10], pix2[11], pix3[10], pix3[11]));
1433         s    += abs(pix1[11] - avg4(pix2[11], pix2[12], pix3[11], pix3[12]));
1434         s    += abs(pix1[12] - avg4(pix2[12], pix2[13], pix3[12], pix3[13]));
1435         s    += abs(pix1[13] - avg4(pix2[13], pix2[14], pix3[13], pix3[14]));
1436         s    += abs(pix1[14] - avg4(pix2[14], pix2[15], pix3[14], pix3[15]));
1437         s    += abs(pix1[15] - avg4(pix2[15], pix2[16], pix3[15], pix3[16]));
1438         pix1 += line_size;
1439         pix2 += line_size;
1440         pix3 += line_size;
1441     }
1442     return s;
1443 }
1444
1445 static inline int pix_abs8_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
1446                              int line_size, int h)
1447 {
1448     int s = 0, i;
1449
1450     for (i = 0; i < h; i++) {
1451         s    += abs(pix1[0] - pix2[0]);
1452         s    += abs(pix1[1] - pix2[1]);
1453         s    += abs(pix1[2] - pix2[2]);
1454         s    += abs(pix1[3] - pix2[3]);
1455         s    += abs(pix1[4] - pix2[4]);
1456         s    += abs(pix1[5] - pix2[5]);
1457         s    += abs(pix1[6] - pix2[6]);
1458         s    += abs(pix1[7] - pix2[7]);
1459         pix1 += line_size;
1460         pix2 += line_size;
1461     }
1462     return s;
1463 }
1464
1465 static int pix_abs8_x2_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
1466                          int line_size, int h)
1467 {
1468     int s = 0, i;
1469
1470     for (i = 0; i < h; i++) {
1471         s    += abs(pix1[0] - avg2(pix2[0], pix2[1]));
1472         s    += abs(pix1[1] - avg2(pix2[1], pix2[2]));
1473         s    += abs(pix1[2] - avg2(pix2[2], pix2[3]));
1474         s    += abs(pix1[3] - avg2(pix2[3], pix2[4]));
1475         s    += abs(pix1[4] - avg2(pix2[4], pix2[5]));
1476         s    += abs(pix1[5] - avg2(pix2[5], pix2[6]));
1477         s    += abs(pix1[6] - avg2(pix2[6], pix2[7]));
1478         s    += abs(pix1[7] - avg2(pix2[7], pix2[8]));
1479         pix1 += line_size;
1480         pix2 += line_size;
1481     }
1482     return s;
1483 }
1484
1485 static int pix_abs8_y2_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
1486                          int line_size, int h)
1487 {
1488     int s = 0, i;
1489     uint8_t *pix3 = pix2 + line_size;
1490
1491     for (i = 0; i < h; i++) {
1492         s    += abs(pix1[0] - avg2(pix2[0], pix3[0]));
1493         s    += abs(pix1[1] - avg2(pix2[1], pix3[1]));
1494         s    += abs(pix1[2] - avg2(pix2[2], pix3[2]));
1495         s    += abs(pix1[3] - avg2(pix2[3], pix3[3]));
1496         s    += abs(pix1[4] - avg2(pix2[4], pix3[4]));
1497         s    += abs(pix1[5] - avg2(pix2[5], pix3[5]));
1498         s    += abs(pix1[6] - avg2(pix2[6], pix3[6]));
1499         s    += abs(pix1[7] - avg2(pix2[7], pix3[7]));
1500         pix1 += line_size;
1501         pix2 += line_size;
1502         pix3 += line_size;
1503     }
1504     return s;
1505 }
1506
1507 static int pix_abs8_xy2_c(MpegEncContext *v, uint8_t *pix1, uint8_t *pix2,
1508                           int line_size, int h)
1509 {
1510     int s = 0, i;
1511     uint8_t *pix3 = pix2 + line_size;
1512
1513     for (i = 0; i < h; i++) {
1514         s    += abs(pix1[0] - avg4(pix2[0], pix2[1], pix3[0], pix3[1]));
1515         s    += abs(pix1[1] - avg4(pix2[1], pix2[2], pix3[1], pix3[2]));
1516         s    += abs(pix1[2] - avg4(pix2[2], pix2[3], pix3[2], pix3[3]));
1517         s    += abs(pix1[3] - avg4(pix2[3], pix2[4], pix3[3], pix3[4]));
1518         s    += abs(pix1[4] - avg4(pix2[4], pix2[5], pix3[4], pix3[5]));
1519         s    += abs(pix1[5] - avg4(pix2[5], pix2[6], pix3[5], pix3[6]));
1520         s    += abs(pix1[6] - avg4(pix2[6], pix2[7], pix3[6], pix3[7]));
1521         s    += abs(pix1[7] - avg4(pix2[7], pix2[8], pix3[7], pix3[8]));
1522         pix1 += line_size;
1523         pix2 += line_size;
1524         pix3 += line_size;
1525     }
1526     return s;
1527 }
1528
1529 static int nsse16_c(MpegEncContext *c, uint8_t *s1, uint8_t *s2, int stride, int h)
1530 {
1531     int score1 = 0, score2 = 0, x, y;
1532
1533     for (y = 0; y < h; y++) {
1534         for (x = 0; x < 16; x++)
1535             score1 += (s1[x] - s2[x]) * (s1[x] - s2[x]);
1536         if (y + 1 < h) {
1537             for (x = 0; x < 15; x++)
1538                 score2 += FFABS(s1[x]     - s1[x + stride] -
1539                                 s1[x + 1] + s1[x + stride + 1]) -
1540                           FFABS(s2[x]     - s2[x + stride] -
1541                                 s2[x + 1] + s2[x + stride + 1]);
1542         }
1543         s1 += stride;
1544         s2 += stride;
1545     }
1546
1547     if (c)
1548         return score1 + FFABS(score2) * c->avctx->nsse_weight;
1549     else
1550         return score1 + FFABS(score2) * 8;
1551 }
1552
1553 static int nsse8_c(MpegEncContext *c, uint8_t *s1, uint8_t *s2, int stride, int h)
1554 {
1555     int score1 = 0, score2 = 0, x, y;
1556
1557     for (y = 0; y < h; y++) {
1558         for (x = 0; x < 8; x++)
1559             score1 += (s1[x] - s2[x]) * (s1[x] - s2[x]);
1560         if (y + 1 < h) {
1561             for (x = 0; x < 7; x++)
1562                 score2 += FFABS(s1[x]     - s1[x + stride] -
1563                                 s1[x + 1] + s1[x + stride + 1]) -
1564                           FFABS(s2[x]     - s2[x + stride] -
1565                                 s2[x + 1] + s2[x + stride + 1]);
1566         }
1567         s1 += stride;
1568         s2 += stride;
1569     }
1570
1571     if (c)
1572         return score1 + FFABS(score2) * c->avctx->nsse_weight;
1573     else
1574         return score1 + FFABS(score2) * 8;
1575 }
1576
1577 static int try_8x8basis_c(int16_t rem[64], int16_t weight[64],
1578                           int16_t basis[64], int scale)
1579 {
1580     int i;
1581     unsigned int sum = 0;
1582
1583     for (i = 0; i < 8 * 8; i++) {
1584         int b = rem[i] + ((basis[i] * scale +
1585                            (1 << (BASIS_SHIFT - RECON_SHIFT - 1))) >>
1586                           (BASIS_SHIFT - RECON_SHIFT));
1587         int w = weight[i];
1588         b >>= RECON_SHIFT;
1589         assert(-512 < b && b < 512);
1590
1591         sum += (w * b) * (w * b) >> 4;
1592     }
1593     return sum >> 2;
1594 }
1595
1596 static void add_8x8basis_c(int16_t rem[64], int16_t basis[64], int scale)
1597 {
1598     int i;
1599
1600     for (i = 0; i < 8 * 8; i++)
1601         rem[i] += (basis[i] * scale +
1602                    (1 << (BASIS_SHIFT - RECON_SHIFT - 1))) >>
1603                   (BASIS_SHIFT - RECON_SHIFT);
1604 }
1605
1606 static int zero_cmp(MpegEncContext *s, uint8_t *a, uint8_t *b,
1607                     int stride, int h)
1608 {
1609     return 0;
1610 }
1611
1612 void ff_set_cmp(DSPContext *c, me_cmp_func *cmp, int type)
1613 {
1614     int i;
1615
1616     memset(cmp, 0, sizeof(void *) * 6);
1617
1618     for (i = 0; i < 6; i++) {
1619         switch (type & 0xFF) {
1620         case FF_CMP_SAD:
1621             cmp[i] = c->sad[i];
1622             break;
1623         case FF_CMP_SATD:
1624             cmp[i] = c->hadamard8_diff[i];
1625             break;
1626         case FF_CMP_SSE:
1627             cmp[i] = c->sse[i];
1628             break;
1629         case FF_CMP_DCT:
1630             cmp[i] = c->dct_sad[i];
1631             break;
1632         case FF_CMP_DCT264:
1633             cmp[i] = c->dct264_sad[i];
1634             break;
1635         case FF_CMP_DCTMAX:
1636             cmp[i] = c->dct_max[i];
1637             break;
1638         case FF_CMP_PSNR:
1639             cmp[i] = c->quant_psnr[i];
1640             break;
1641         case FF_CMP_BIT:
1642             cmp[i] = c->bit[i];
1643             break;
1644         case FF_CMP_RD:
1645             cmp[i] = c->rd[i];
1646             break;
1647         case FF_CMP_VSAD:
1648             cmp[i] = c->vsad[i];
1649             break;
1650         case FF_CMP_VSSE:
1651             cmp[i] = c->vsse[i];
1652             break;
1653         case FF_CMP_ZERO:
1654             cmp[i] = zero_cmp;
1655             break;
1656         case FF_CMP_NSSE:
1657             cmp[i] = c->nsse[i];
1658             break;
1659         default:
1660             av_log(NULL, AV_LOG_ERROR,
1661                    "internal error in cmp function selection\n");
1662         }
1663     }
1664 }
1665
1666 static void diff_bytes_c(uint8_t *dst, uint8_t *src1, uint8_t *src2, int w)
1667 {
1668     long i;
1669
1670 #if !HAVE_FAST_UNALIGNED
1671     if ((long) src2 & (sizeof(long) - 1)) {
1672         for (i = 0; i + 7 < w; i += 8) {
1673             dst[i + 0] = src1[i + 0] - src2[i + 0];
1674             dst[i + 1] = src1[i + 1] - src2[i + 1];
1675             dst[i + 2] = src1[i + 2] - src2[i + 2];
1676             dst[i + 3] = src1[i + 3] - src2[i + 3];
1677             dst[i + 4] = src1[i + 4] - src2[i + 4];
1678             dst[i + 5] = src1[i + 5] - src2[i + 5];
1679             dst[i + 6] = src1[i + 6] - src2[i + 6];
1680             dst[i + 7] = src1[i + 7] - src2[i + 7];
1681         }
1682     } else
1683 #endif
1684     for (i = 0; i <= w - (int) sizeof(long); i += sizeof(long)) {
1685         long a = *(long *) (src1 + i);
1686         long b = *(long *) (src2 + i);
1687         *(long *) (dst + i) = ((a | pb_80) - (b & pb_7f)) ^
1688                               ((a ^ b ^ pb_80) & pb_80);
1689     }
1690     for (; i < w; i++)
1691         dst[i + 0] = src1[i + 0] - src2[i + 0];
1692 }
1693
1694 static void sub_hfyu_median_prediction_c(uint8_t *dst, const uint8_t *src1,
1695                                          const uint8_t *src2, int w,
1696                                          int *left, int *left_top)
1697 {
1698     int i;
1699     uint8_t l, lt;
1700
1701     l  = *left;
1702     lt = *left_top;
1703
1704     for (i = 0; i < w; i++) {
1705         const int pred = mid_pred(l, src1[i], (l + src1[i] - lt) & 0xFF);
1706         lt     = src1[i];
1707         l      = src2[i];
1708         dst[i] = l - pred;
1709     }
1710
1711     *left     = l;
1712     *left_top = lt;
1713 }
1714
1715 #define BUTTERFLY2(o1, o2, i1, i2)              \
1716     o1 = (i1) + (i2);                           \
1717     o2 = (i1) - (i2);
1718
1719 #define BUTTERFLY1(x, y)                        \
1720     {                                           \
1721         int a, b;                               \
1722         a = x;                                  \
1723         b = y;                                  \
1724         x = a + b;                              \
1725         y = a - b;                              \
1726     }
1727
1728 #define BUTTERFLYA(x, y) (FFABS((x) + (y)) + FFABS((x) - (y)))
1729
1730 static int hadamard8_diff8x8_c(MpegEncContext *s, uint8_t *dst,
1731                                uint8_t *src, int stride, int h)
1732 {
1733     int i, temp[64], sum = 0;
1734
1735     assert(h == 8);
1736
1737     for (i = 0; i < 8; i++) {
1738         // FIXME: try pointer walks
1739         BUTTERFLY2(temp[8 * i + 0], temp[8 * i + 1],
1740                    src[stride * i + 0] - dst[stride * i + 0],
1741                    src[stride * i + 1] - dst[stride * i + 1]);
1742         BUTTERFLY2(temp[8 * i + 2], temp[8 * i + 3],
1743                    src[stride * i + 2] - dst[stride * i + 2],
1744                    src[stride * i + 3] - dst[stride * i + 3]);
1745         BUTTERFLY2(temp[8 * i + 4], temp[8 * i + 5],
1746                    src[stride * i + 4] - dst[stride * i + 4],
1747                    src[stride * i + 5] - dst[stride * i + 5]);
1748         BUTTERFLY2(temp[8 * i + 6], temp[8 * i + 7],
1749                    src[stride * i + 6] - dst[stride * i + 6],
1750                    src[stride * i + 7] - dst[stride * i + 7]);
1751
1752         BUTTERFLY1(temp[8 * i + 0], temp[8 * i + 2]);
1753         BUTTERFLY1(temp[8 * i + 1], temp[8 * i + 3]);
1754         BUTTERFLY1(temp[8 * i + 4], temp[8 * i + 6]);
1755         BUTTERFLY1(temp[8 * i + 5], temp[8 * i + 7]);
1756
1757         BUTTERFLY1(temp[8 * i + 0], temp[8 * i + 4]);
1758         BUTTERFLY1(temp[8 * i + 1], temp[8 * i + 5]);
1759         BUTTERFLY1(temp[8 * i + 2], temp[8 * i + 6]);
1760         BUTTERFLY1(temp[8 * i + 3], temp[8 * i + 7]);
1761     }
1762
1763     for (i = 0; i < 8; i++) {
1764         BUTTERFLY1(temp[8 * 0 + i], temp[8 * 1 + i]);
1765         BUTTERFLY1(temp[8 * 2 + i], temp[8 * 3 + i]);
1766         BUTTERFLY1(temp[8 * 4 + i], temp[8 * 5 + i]);
1767         BUTTERFLY1(temp[8 * 6 + i], temp[8 * 7 + i]);
1768
1769         BUTTERFLY1(temp[8 * 0 + i], temp[8 * 2 + i]);
1770         BUTTERFLY1(temp[8 * 1 + i], temp[8 * 3 + i]);
1771         BUTTERFLY1(temp[8 * 4 + i], temp[8 * 6 + i]);
1772         BUTTERFLY1(temp[8 * 5 + i], temp[8 * 7 + i]);
1773
1774         sum += BUTTERFLYA(temp[8 * 0 + i], temp[8 * 4 + i]) +
1775                BUTTERFLYA(temp[8 * 1 + i], temp[8 * 5 + i]) +
1776                BUTTERFLYA(temp[8 * 2 + i], temp[8 * 6 + i]) +
1777                BUTTERFLYA(temp[8 * 3 + i], temp[8 * 7 + i]);
1778     }
1779     return sum;
1780 }
1781
1782 static int hadamard8_intra8x8_c(MpegEncContext *s, uint8_t *src,
1783                                 uint8_t *dummy, int stride, int h)
1784 {
1785     int i, temp[64], sum = 0;
1786
1787     assert(h == 8);
1788
1789     for (i = 0; i < 8; i++) {
1790         // FIXME: try pointer walks
1791         BUTTERFLY2(temp[8 * i + 0], temp[8 * i + 1],
1792                    src[stride * i + 0], src[stride * i + 1]);
1793         BUTTERFLY2(temp[8 * i + 2], temp[8 * i + 3],
1794                    src[stride * i + 2], src[stride * i + 3]);
1795         BUTTERFLY2(temp[8 * i + 4], temp[8 * i + 5],
1796                    src[stride * i + 4], src[stride * i + 5]);
1797         BUTTERFLY2(temp[8 * i + 6], temp[8 * i + 7],
1798                    src[stride * i + 6], src[stride * i + 7]);
1799
1800         BUTTERFLY1(temp[8 * i + 0], temp[8 * i + 2]);
1801         BUTTERFLY1(temp[8 * i + 1], temp[8 * i + 3]);
1802         BUTTERFLY1(temp[8 * i + 4], temp[8 * i + 6]);
1803         BUTTERFLY1(temp[8 * i + 5], temp[8 * i + 7]);
1804
1805         BUTTERFLY1(temp[8 * i + 0], temp[8 * i + 4]);
1806         BUTTERFLY1(temp[8 * i + 1], temp[8 * i + 5]);
1807         BUTTERFLY1(temp[8 * i + 2], temp[8 * i + 6]);
1808         BUTTERFLY1(temp[8 * i + 3], temp[8 * i + 7]);
1809     }
1810
1811     for (i = 0; i < 8; i++) {
1812         BUTTERFLY1(temp[8 * 0 + i], temp[8 * 1 + i]);
1813         BUTTERFLY1(temp[8 * 2 + i], temp[8 * 3 + i]);
1814         BUTTERFLY1(temp[8 * 4 + i], temp[8 * 5 + i]);
1815         BUTTERFLY1(temp[8 * 6 + i], temp[8 * 7 + i]);
1816
1817         BUTTERFLY1(temp[8 * 0 + i], temp[8 * 2 + i]);
1818         BUTTERFLY1(temp[8 * 1 + i], temp[8 * 3 + i]);
1819         BUTTERFLY1(temp[8 * 4 + i], temp[8 * 6 + i]);
1820         BUTTERFLY1(temp[8 * 5 + i], temp[8 * 7 + i]);
1821
1822         sum +=
1823             BUTTERFLYA(temp[8 * 0 + i], temp[8 * 4 + i])
1824             + BUTTERFLYA(temp[8 * 1 + i], temp[8 * 5 + i])
1825             + BUTTERFLYA(temp[8 * 2 + i], temp[8 * 6 + i])
1826             + BUTTERFLYA(temp[8 * 3 + i], temp[8 * 7 + i]);
1827     }
1828
1829     sum -= FFABS(temp[8 * 0] + temp[8 * 4]); // -mean
1830
1831     return sum;
1832 }
1833
1834 static int dct_sad8x8_c(MpegEncContext *s, uint8_t *src1,
1835                         uint8_t *src2, int stride, int h)
1836 {
1837     LOCAL_ALIGNED_16(int16_t, temp, [64]);
1838
1839     assert(h == 8);
1840
1841     s->dsp.diff_pixels(temp, src1, src2, stride);
1842     s->dsp.fdct(temp);
1843     return s->dsp.sum_abs_dctelem(temp);
1844 }
1845
1846 #if CONFIG_GPL
1847 #define DCT8_1D                                         \
1848     {                                                   \
1849         const int s07 = SRC(0) + SRC(7);                \
1850         const int s16 = SRC(1) + SRC(6);                \
1851         const int s25 = SRC(2) + SRC(5);                \
1852         const int s34 = SRC(3) + SRC(4);                \
1853         const int a0  = s07 + s34;                      \
1854         const int a1  = s16 + s25;                      \
1855         const int a2  = s07 - s34;                      \
1856         const int a3  = s16 - s25;                      \
1857         const int d07 = SRC(0) - SRC(7);                \
1858         const int d16 = SRC(1) - SRC(6);                \
1859         const int d25 = SRC(2) - SRC(5);                \
1860         const int d34 = SRC(3) - SRC(4);                \
1861         const int a4  = d16 + d25 + (d07 + (d07 >> 1)); \
1862         const int a5  = d07 - d34 - (d25 + (d25 >> 1)); \
1863         const int a6  = d07 + d34 - (d16 + (d16 >> 1)); \
1864         const int a7  = d16 - d25 + (d34 + (d34 >> 1)); \
1865         DST(0, a0 + a1);                                \
1866         DST(1, a4 + (a7 >> 2));                         \
1867         DST(2, a2 + (a3 >> 1));                         \
1868         DST(3, a5 + (a6 >> 2));                         \
1869         DST(4, a0 - a1);                                \
1870         DST(5, a6 - (a5 >> 2));                         \
1871         DST(6, (a2 >> 1) - a3);                         \
1872         DST(7, (a4 >> 2) - a7);                         \
1873     }
1874
1875 static int dct264_sad8x8_c(MpegEncContext *s, uint8_t *src1,
1876                            uint8_t *src2, int stride, int h)
1877 {
1878     int16_t dct[8][8];
1879     int i, sum = 0;
1880
1881     s->dsp.diff_pixels(dct[0], src1, src2, stride);
1882
1883 #define SRC(x) dct[i][x]
1884 #define DST(x, v) dct[i][x] = v
1885     for (i = 0; i < 8; i++)
1886         DCT8_1D
1887 #undef SRC
1888 #undef DST
1889
1890 #define SRC(x) dct[x][i]
1891 #define DST(x, v) sum += FFABS(v)
1892         for (i = 0; i < 8; i++)
1893             DCT8_1D
1894 #undef SRC
1895 #undef DST
1896             return sum;
1897 }
1898 #endif
1899
1900 static int dct_max8x8_c(MpegEncContext *s, uint8_t *src1,
1901                         uint8_t *src2, int stride, int h)
1902 {
1903     LOCAL_ALIGNED_16(int16_t, temp, [64]);
1904     int sum = 0, i;
1905
1906     assert(h == 8);
1907
1908     s->dsp.diff_pixels(temp, src1, src2, stride);
1909     s->dsp.fdct(temp);
1910
1911     for (i = 0; i < 64; i++)
1912         sum = FFMAX(sum, FFABS(temp[i]));
1913
1914     return sum;
1915 }
1916
1917 static int quant_psnr8x8_c(MpegEncContext *s, uint8_t *src1,
1918                            uint8_t *src2, int stride, int h)
1919 {
1920     LOCAL_ALIGNED_16(int16_t, temp, [64 * 2]);
1921     int16_t *const bak = temp + 64;
1922     int sum = 0, i;
1923
1924     assert(h == 8);
1925     s->mb_intra = 0;
1926
1927     s->dsp.diff_pixels(temp, src1, src2, stride);
1928
1929     memcpy(bak, temp, 64 * sizeof(int16_t));
1930
1931     s->block_last_index[0 /* FIXME */] =
1932         s->fast_dct_quantize(s, temp, 0 /* FIXME */, s->qscale, &i);
1933     s->dct_unquantize_inter(s, temp, 0, s->qscale);
1934     ff_simple_idct_8(temp); // FIXME
1935
1936     for (i = 0; i < 64; i++)
1937         sum += (temp[i] - bak[i]) * (temp[i] - bak[i]);
1938
1939     return sum;
1940 }
1941
1942 static int rd8x8_c(MpegEncContext *s, uint8_t *src1, uint8_t *src2,
1943                    int stride, int h)
1944 {
1945     const uint8_t *scantable = s->intra_scantable.permutated;
1946     LOCAL_ALIGNED_16(int16_t, temp, [64]);
1947     LOCAL_ALIGNED_16(uint8_t, lsrc1, [64]);
1948     LOCAL_ALIGNED_16(uint8_t, lsrc2, [64]);
1949     int i, last, run, bits, level, distortion, start_i;
1950     const int esc_length = s->ac_esc_length;
1951     uint8_t *length, *last_length;
1952
1953     assert(h == 8);
1954
1955     copy_block8(lsrc1, src1, 8, stride, 8);
1956     copy_block8(lsrc2, src2, 8, stride, 8);
1957
1958     s->dsp.diff_pixels(temp, lsrc1, lsrc2, 8);
1959
1960     s->block_last_index[0 /* FIXME */] =
1961     last                               =
1962         s->fast_dct_quantize(s, temp, 0 /* FIXME */, s->qscale, &i);
1963
1964     bits = 0;
1965
1966     if (s->mb_intra) {
1967         start_i     = 1;
1968         length      = s->intra_ac_vlc_length;
1969         last_length = s->intra_ac_vlc_last_length;
1970         bits       += s->luma_dc_vlc_length[temp[0] + 256]; // FIXME: chroma
1971     } else {
1972         start_i     = 0;
1973         length      = s->inter_ac_vlc_length;
1974         last_length = s->inter_ac_vlc_last_length;
1975     }
1976
1977     if (last >= start_i) {
1978         run = 0;
1979         for (i = start_i; i < last; i++) {
1980             int j = scantable[i];
1981             level = temp[j];
1982
1983             if (level) {
1984                 level += 64;
1985                 if ((level & (~127)) == 0)
1986                     bits += length[UNI_AC_ENC_INDEX(run, level)];
1987                 else
1988                     bits += esc_length;
1989                 run = 0;
1990             } else
1991                 run++;
1992         }
1993         i = scantable[last];
1994
1995         level = temp[i] + 64;
1996
1997         assert(level - 64);
1998
1999         if ((level & (~127)) == 0) {
2000             bits += last_length[UNI_AC_ENC_INDEX(run, level)];
2001         } else
2002             bits += esc_length;
2003     }
2004
2005     if (last >= 0) {
2006         if (s->mb_intra)
2007             s->dct_unquantize_intra(s, temp, 0, s->qscale);
2008         else
2009             s->dct_unquantize_inter(s, temp, 0, s->qscale);
2010     }
2011
2012     s->dsp.idct_add(lsrc2, 8, temp);
2013
2014     distortion = s->dsp.sse[1](NULL, lsrc2, lsrc1, 8, 8);
2015
2016     return distortion + ((bits * s->qscale * s->qscale * 109 + 64) >> 7);
2017 }
2018
2019 static int bit8x8_c(MpegEncContext *s, uint8_t *src1, uint8_t *src2,
2020                     int stride, int h)
2021 {
2022     const uint8_t *scantable = s->intra_scantable.permutated;
2023     LOCAL_ALIGNED_16(int16_t, temp, [64]);
2024     int i, last, run, bits, level, start_i;
2025     const int esc_length = s->ac_esc_length;
2026     uint8_t *length, *last_length;
2027
2028     assert(h == 8);
2029
2030     s->dsp.diff_pixels(temp, src1, src2, stride);
2031
2032     s->block_last_index[0 /* FIXME */] =
2033     last                               =
2034         s->fast_dct_quantize(s, temp, 0 /* FIXME */, s->qscale, &i);
2035
2036     bits = 0;
2037
2038     if (s->mb_intra) {
2039         start_i     = 1;
2040         length      = s->intra_ac_vlc_length;
2041         last_length = s->intra_ac_vlc_last_length;
2042         bits       += s->luma_dc_vlc_length[temp[0] + 256]; // FIXME: chroma
2043     } else {
2044         start_i     = 0;
2045         length      = s->inter_ac_vlc_length;
2046         last_length = s->inter_ac_vlc_last_length;
2047     }
2048
2049     if (last >= start_i) {
2050         run = 0;
2051         for (i = start_i; i < last; i++) {
2052             int j = scantable[i];
2053             level = temp[j];
2054
2055             if (level) {
2056                 level += 64;
2057                 if ((level & (~127)) == 0)
2058                     bits += length[UNI_AC_ENC_INDEX(run, level)];
2059                 else
2060                     bits += esc_length;
2061                 run = 0;
2062             } else
2063                 run++;
2064         }
2065         i = scantable[last];
2066
2067         level = temp[i] + 64;
2068
2069         assert(level - 64);
2070
2071         if ((level & (~127)) == 0)
2072             bits += last_length[UNI_AC_ENC_INDEX(run, level)];
2073         else
2074             bits += esc_length;
2075     }
2076
2077     return bits;
2078 }
2079
2080 #define VSAD_INTRA(size)                                                \
2081 static int vsad_intra ## size ## _c(MpegEncContext *c,                  \
2082                                     uint8_t *s, uint8_t *dummy,         \
2083                                     int stride, int h)                  \
2084 {                                                                       \
2085     int score = 0, x, y;                                                \
2086                                                                         \
2087     for (y = 1; y < h; y++) {                                           \
2088         for (x = 0; x < size; x += 4) {                                 \
2089             score += FFABS(s[x]     - s[x + stride])     +              \
2090                      FFABS(s[x + 1] - s[x + stride + 1]) +              \
2091                      FFABS(s[x + 2] - s[x + 2 + stride]) +              \
2092                      FFABS(s[x + 3] - s[x + 3 + stride]);               \
2093         }                                                               \
2094         s += stride;                                                    \
2095     }                                                                   \
2096                                                                         \
2097     return score;                                                       \
2098 }
2099 VSAD_INTRA(8)
2100 VSAD_INTRA(16)
2101
2102 static int vsad16_c(MpegEncContext *c, uint8_t *s1, uint8_t *s2,
2103                     int stride, int h)
2104 {
2105     int score = 0, x, y;
2106
2107     for (y = 1; y < h; y++) {
2108         for (x = 0; x < 16; x++)
2109             score += FFABS(s1[x] - s2[x] - s1[x + stride] + s2[x + stride]);
2110         s1 += stride;
2111         s2 += stride;
2112     }
2113
2114     return score;
2115 }
2116
2117 #define SQ(a) ((a) * (a))
2118 #define VSSE_INTRA(size)                                                \
2119 static int vsse_intra ## size ## _c(MpegEncContext *c,                  \
2120                                     uint8_t *s, uint8_t *dummy,         \
2121                                     int stride, int h)                  \
2122 {                                                                       \
2123     int score = 0, x, y;                                                \
2124                                                                         \
2125     for (y = 1; y < h; y++) {                                           \
2126         for (x = 0; x < size; x += 4) {                                 \
2127             score += SQ(s[x]     - s[x + stride]) +                     \
2128                      SQ(s[x + 1] - s[x + stride + 1]) +                 \
2129                      SQ(s[x + 2] - s[x + stride + 2]) +                 \
2130                      SQ(s[x + 3] - s[x + stride + 3]);                  \
2131         }                                                               \
2132         s += stride;                                                    \
2133     }                                                                   \
2134                                                                         \
2135     return score;                                                       \
2136 }
2137 VSSE_INTRA(8)
2138 VSSE_INTRA(16)
2139
2140 static int vsse16_c(MpegEncContext *c, uint8_t *s1, uint8_t *s2,
2141                     int stride, int h)
2142 {
2143     int score = 0, x, y;
2144
2145     for (y = 1; y < h; y++) {
2146         for (x = 0; x < 16; x++)
2147             score += SQ(s1[x] - s2[x] - s1[x + stride] + s2[x + stride]);
2148         s1 += stride;
2149         s2 += stride;
2150     }
2151
2152     return score;
2153 }
2154
2155 static int ssd_int8_vs_int16_c(const int8_t *pix1, const int16_t *pix2,
2156                                int size)
2157 {
2158     int score = 0, i;
2159
2160     for (i = 0; i < size; i++)
2161         score += (pix1[i] - pix2[i]) * (pix1[i] - pix2[i]);
2162     return score;
2163 }
2164
2165 #define WRAPPER8_16_SQ(name8, name16)                                   \
2166 static int name16(MpegEncContext *s, uint8_t *dst, uint8_t *src,        \
2167                   int stride, int h)                                    \
2168 {                                                                       \
2169     int score = 0;                                                      \
2170                                                                         \
2171     score += name8(s, dst, src, stride, 8);                             \
2172     score += name8(s, dst + 8, src + 8, stride, 8);                     \
2173     if (h == 16) {                                                      \
2174         dst   += 8 * stride;                                            \
2175         src   += 8 * stride;                                            \
2176         score += name8(s, dst, src, stride, 8);                         \
2177         score += name8(s, dst + 8, src + 8, stride, 8);                 \
2178     }                                                                   \
2179     return score;                                                       \
2180 }
2181
2182 WRAPPER8_16_SQ(hadamard8_diff8x8_c, hadamard8_diff16_c)
2183 WRAPPER8_16_SQ(hadamard8_intra8x8_c, hadamard8_intra16_c)
2184 WRAPPER8_16_SQ(dct_sad8x8_c, dct_sad16_c)
2185 #if CONFIG_GPL
2186 WRAPPER8_16_SQ(dct264_sad8x8_c, dct264_sad16_c)
2187 #endif
2188 WRAPPER8_16_SQ(dct_max8x8_c, dct_max16_c)
2189 WRAPPER8_16_SQ(quant_psnr8x8_c, quant_psnr16_c)
2190 WRAPPER8_16_SQ(rd8x8_c, rd16_c)
2191 WRAPPER8_16_SQ(bit8x8_c, bit16_c)
2192
2193 static inline uint32_t clipf_c_one(uint32_t a, uint32_t mini,
2194                                    uint32_t maxi, uint32_t maxisign)
2195 {
2196     if (a > mini)
2197         return mini;
2198     else if ((a ^ (1U << 31)) > maxisign)
2199         return maxi;
2200     else
2201         return a;
2202 }
2203
2204 static void vector_clipf_c_opposite_sign(float *dst, const float *src,
2205                                          float *min, float *max, int len)
2206 {
2207     int i;
2208     uint32_t mini        = *(uint32_t *) min;
2209     uint32_t maxi        = *(uint32_t *) max;
2210     uint32_t maxisign    = maxi ^ (1U << 31);
2211     uint32_t *dsti       = (uint32_t *) dst;
2212     const uint32_t *srci = (const uint32_t *) src;
2213
2214     for (i = 0; i < len; i += 8) {
2215         dsti[i + 0] = clipf_c_one(srci[i + 0], mini, maxi, maxisign);
2216         dsti[i + 1] = clipf_c_one(srci[i + 1], mini, maxi, maxisign);
2217         dsti[i + 2] = clipf_c_one(srci[i + 2], mini, maxi, maxisign);
2218         dsti[i + 3] = clipf_c_one(srci[i + 3], mini, maxi, maxisign);
2219         dsti[i + 4] = clipf_c_one(srci[i + 4], mini, maxi, maxisign);
2220         dsti[i + 5] = clipf_c_one(srci[i + 5], mini, maxi, maxisign);
2221         dsti[i + 6] = clipf_c_one(srci[i + 6], mini, maxi, maxisign);
2222         dsti[i + 7] = clipf_c_one(srci[i + 7], mini, maxi, maxisign);
2223     }
2224 }
2225
2226 static void vector_clipf_c(float *dst, const float *src,
2227                            float min, float max, int len)
2228 {
2229     int i;
2230
2231     if (min < 0 && max > 0) {
2232         vector_clipf_c_opposite_sign(dst, src, &min, &max, len);
2233     } else {
2234         for (i = 0; i < len; i += 8) {
2235             dst[i]     = av_clipf(src[i], min, max);
2236             dst[i + 1] = av_clipf(src[i + 1], min, max);
2237             dst[i + 2] = av_clipf(src[i + 2], min, max);
2238             dst[i + 3] = av_clipf(src[i + 3], min, max);
2239             dst[i + 4] = av_clipf(src[i + 4], min, max);
2240             dst[i + 5] = av_clipf(src[i + 5], min, max);
2241             dst[i + 6] = av_clipf(src[i + 6], min, max);
2242             dst[i + 7] = av_clipf(src[i + 7], min, max);
2243         }
2244     }
2245 }
2246
2247 static int32_t scalarproduct_int16_c(const int16_t *v1, const int16_t *v2,
2248                                      int order)
2249 {
2250     int res = 0;
2251
2252     while (order--)
2253         res += *v1++ **v2++;
2254
2255     return res;
2256 }
2257
2258 static int32_t scalarproduct_and_madd_int16_c(int16_t *v1, const int16_t *v2,
2259                                               const int16_t *v3,
2260                                               int order, int mul)
2261 {
2262     int res = 0;
2263
2264     while (order--) {
2265         res   += *v1 * *v2++;
2266         *v1++ += mul * *v3++;
2267     }
2268     return res;
2269 }
2270
2271 static void vector_clip_int32_c(int32_t *dst, const int32_t *src, int32_t min,
2272                                 int32_t max, unsigned int len)
2273 {
2274     do {
2275         *dst++ = av_clip(*src++, min, max);
2276         *dst++ = av_clip(*src++, min, max);
2277         *dst++ = av_clip(*src++, min, max);
2278         *dst++ = av_clip(*src++, min, max);
2279         *dst++ = av_clip(*src++, min, max);
2280         *dst++ = av_clip(*src++, min, max);
2281         *dst++ = av_clip(*src++, min, max);
2282         *dst++ = av_clip(*src++, min, max);
2283         len   -= 8;
2284     } while (len > 0);
2285 }
2286
2287 static void jref_idct_put(uint8_t *dest, int line_size, int16_t *block)
2288 {
2289     ff_j_rev_dct(block);
2290     put_pixels_clamped_c(block, dest, line_size);
2291 }
2292
2293 static void jref_idct_add(uint8_t *dest, int line_size, int16_t *block)
2294 {
2295     ff_j_rev_dct(block);
2296     add_pixels_clamped_c(block, dest, line_size);
2297 }
2298
2299 /* draw the edges of width 'w' of an image of size width, height */
2300 // FIXME: Check that this is OK for MPEG-4 interlaced.
2301 static void draw_edges_8_c(uint8_t *buf, int wrap, int width, int height,
2302                            int w, int h, int sides)
2303 {
2304     uint8_t *ptr = buf, *last_line;
2305     int i;
2306
2307     /* left and right */
2308     for (i = 0; i < height; i++) {
2309         memset(ptr - w, ptr[0], w);
2310         memset(ptr + width, ptr[width - 1], w);
2311         ptr += wrap;
2312     }
2313
2314     /* top and bottom + corners */
2315     buf -= w;
2316     last_line = buf + (height - 1) * wrap;
2317     if (sides & EDGE_TOP)
2318         for (i = 0; i < h; i++)
2319             // top
2320             memcpy(buf - (i + 1) * wrap, buf, width + w + w);
2321     if (sides & EDGE_BOTTOM)
2322         for (i = 0; i < h; i++)
2323             // bottom
2324             memcpy(last_line + (i + 1) * wrap, last_line, width + w + w);
2325 }
2326
2327 static void clear_block_8_c(int16_t *block)
2328 {
2329     memset(block, 0, sizeof(int16_t) * 64);
2330 }
2331
2332 static void clear_blocks_8_c(int16_t *blocks)
2333 {
2334     memset(blocks, 0, sizeof(int16_t) * 6 * 64);
2335 }
2336
2337 /* init static data */
2338 av_cold void ff_dsputil_static_init(void)
2339 {
2340     int i;
2341
2342     for (i = 0; i < 512; i++)
2343         ff_square_tab[i] = (i - 256) * (i - 256);
2344 }
2345
2346 av_cold void ff_dsputil_init(DSPContext *c, AVCodecContext *avctx)
2347 {
2348     const unsigned high_bit_depth = avctx->bits_per_raw_sample > 8;
2349
2350 #if CONFIG_ENCODERS
2351     if (avctx->bits_per_raw_sample == 10) {
2352         c->fdct    = ff_jpeg_fdct_islow_10;
2353         c->fdct248 = ff_fdct248_islow_10;
2354     } else {
2355         if (avctx->dct_algo == FF_DCT_FASTINT) {
2356             c->fdct    = ff_fdct_ifast;
2357             c->fdct248 = ff_fdct_ifast248;
2358         } else if (avctx->dct_algo == FF_DCT_FAAN) {
2359             c->fdct    = ff_faandct;
2360             c->fdct248 = ff_faandct248;
2361         } else {
2362             c->fdct    = ff_jpeg_fdct_islow_8; // slow/accurate/default
2363             c->fdct248 = ff_fdct248_islow_8;
2364         }
2365     }
2366 #endif /* CONFIG_ENCODERS */
2367
2368     if (avctx->bits_per_raw_sample == 10) {
2369         c->idct_put              = ff_simple_idct_put_10;
2370         c->idct_add              = ff_simple_idct_add_10;
2371         c->idct                  = ff_simple_idct_10;
2372         c->idct_permutation_type = FF_NO_IDCT_PERM;
2373     } else {
2374         if (avctx->idct_algo == FF_IDCT_INT) {
2375             c->idct_put              = jref_idct_put;
2376             c->idct_add              = jref_idct_add;
2377             c->idct                  = ff_j_rev_dct;
2378             c->idct_permutation_type = FF_LIBMPEG2_IDCT_PERM;
2379         } else if (avctx->idct_algo == FF_IDCT_FAAN) {
2380             c->idct_put              = ff_faanidct_put;
2381             c->idct_add              = ff_faanidct_add;
2382             c->idct                  = ff_faanidct;
2383             c->idct_permutation_type = FF_NO_IDCT_PERM;
2384         } else { // accurate/default
2385             c->idct_put              = ff_simple_idct_put_8;
2386             c->idct_add              = ff_simple_idct_add_8;
2387             c->idct                  = ff_simple_idct_8;
2388             c->idct_permutation_type = FF_NO_IDCT_PERM;
2389         }
2390     }
2391
2392     c->diff_pixels = diff_pixels_c;
2393
2394     c->put_pixels_clamped        = put_pixels_clamped_c;
2395     c->put_signed_pixels_clamped = put_signed_pixels_clamped_c;
2396     c->add_pixels_clamped        = add_pixels_clamped_c;
2397
2398     c->sum_abs_dctelem = sum_abs_dctelem_c;
2399
2400     c->gmc1 = gmc1_c;
2401     c->gmc  = ff_gmc_c;
2402
2403     c->pix_sum   = pix_sum_c;
2404     c->pix_norm1 = pix_norm1_c;
2405
2406     c->fill_block_tab[0] = fill_block16_c;
2407     c->fill_block_tab[1] = fill_block8_c;
2408
2409     /* TODO [0] 16  [1] 8 */
2410     c->pix_abs[0][0] = pix_abs16_c;
2411     c->pix_abs[0][1] = pix_abs16_x2_c;
2412     c->pix_abs[0][2] = pix_abs16_y2_c;
2413     c->pix_abs[0][3] = pix_abs16_xy2_c;
2414     c->pix_abs[1][0] = pix_abs8_c;
2415     c->pix_abs[1][1] = pix_abs8_x2_c;
2416     c->pix_abs[1][2] = pix_abs8_y2_c;
2417     c->pix_abs[1][3] = pix_abs8_xy2_c;
2418
2419 #define dspfunc(PFX, IDX, NUM)                              \
2420     c->PFX ## _pixels_tab[IDX][0]  = PFX ## NUM ## _mc00_c; \
2421     c->PFX ## _pixels_tab[IDX][1]  = PFX ## NUM ## _mc10_c; \
2422     c->PFX ## _pixels_tab[IDX][2]  = PFX ## NUM ## _mc20_c; \
2423     c->PFX ## _pixels_tab[IDX][3]  = PFX ## NUM ## _mc30_c; \
2424     c->PFX ## _pixels_tab[IDX][4]  = PFX ## NUM ## _mc01_c; \
2425     c->PFX ## _pixels_tab[IDX][5]  = PFX ## NUM ## _mc11_c; \
2426     c->PFX ## _pixels_tab[IDX][6]  = PFX ## NUM ## _mc21_c; \
2427     c->PFX ## _pixels_tab[IDX][7]  = PFX ## NUM ## _mc31_c; \
2428     c->PFX ## _pixels_tab[IDX][8]  = PFX ## NUM ## _mc02_c; \
2429     c->PFX ## _pixels_tab[IDX][9]  = PFX ## NUM ## _mc12_c; \
2430     c->PFX ## _pixels_tab[IDX][10] = PFX ## NUM ## _mc22_c; \
2431     c->PFX ## _pixels_tab[IDX][11] = PFX ## NUM ## _mc32_c; \
2432     c->PFX ## _pixels_tab[IDX][12] = PFX ## NUM ## _mc03_c; \
2433     c->PFX ## _pixels_tab[IDX][13] = PFX ## NUM ## _mc13_c; \
2434     c->PFX ## _pixels_tab[IDX][14] = PFX ## NUM ## _mc23_c; \
2435     c->PFX ## _pixels_tab[IDX][15] = PFX ## NUM ## _mc33_c
2436
2437     dspfunc(put_qpel, 0, 16);
2438     dspfunc(put_qpel, 1, 8);
2439
2440     dspfunc(put_no_rnd_qpel, 0, 16);
2441     dspfunc(put_no_rnd_qpel, 1, 8);
2442
2443     dspfunc(avg_qpel, 0, 16);
2444     dspfunc(avg_qpel, 1, 8);
2445
2446 #undef dspfunc
2447
2448     c->put_mspel_pixels_tab[0] = ff_put_pixels8x8_c;
2449     c->put_mspel_pixels_tab[1] = put_mspel8_mc10_c;
2450     c->put_mspel_pixels_tab[2] = put_mspel8_mc20_c;
2451     c->put_mspel_pixels_tab[3] = put_mspel8_mc30_c;
2452     c->put_mspel_pixels_tab[4] = put_mspel8_mc02_c;
2453     c->put_mspel_pixels_tab[5] = put_mspel8_mc12_c;
2454     c->put_mspel_pixels_tab[6] = put_mspel8_mc22_c;
2455     c->put_mspel_pixels_tab[7] = put_mspel8_mc32_c;
2456
2457 #define SET_CMP_FUNC(name)                      \
2458     c->name[0] = name ## 16_c;                  \
2459     c->name[1] = name ## 8x8_c;
2460
2461     SET_CMP_FUNC(hadamard8_diff)
2462     c->hadamard8_diff[4] = hadamard8_intra16_c;
2463     c->hadamard8_diff[5] = hadamard8_intra8x8_c;
2464     SET_CMP_FUNC(dct_sad)
2465     SET_CMP_FUNC(dct_max)
2466 #if CONFIG_GPL
2467     SET_CMP_FUNC(dct264_sad)
2468 #endif
2469     c->sad[0] = pix_abs16_c;
2470     c->sad[1] = pix_abs8_c;
2471     c->sse[0] = sse16_c;
2472     c->sse[1] = sse8_c;
2473     c->sse[2] = sse4_c;
2474     SET_CMP_FUNC(quant_psnr)
2475     SET_CMP_FUNC(rd)
2476     SET_CMP_FUNC(bit)
2477     c->vsad[0] = vsad16_c;
2478     c->vsad[4] = vsad_intra16_c;
2479     c->vsad[5] = vsad_intra8_c;
2480     c->vsse[0] = vsse16_c;
2481     c->vsse[4] = vsse_intra16_c;
2482     c->vsse[5] = vsse_intra8_c;
2483     c->nsse[0] = nsse16_c;
2484     c->nsse[1] = nsse8_c;
2485
2486     c->ssd_int8_vs_int16 = ssd_int8_vs_int16_c;
2487
2488     c->diff_bytes                 = diff_bytes_c;
2489     c->sub_hfyu_median_prediction = sub_hfyu_median_prediction_c;
2490
2491     c->bswap_buf   = bswap_buf;
2492     c->bswap16_buf = bswap16_buf;
2493
2494     c->try_8x8basis = try_8x8basis_c;
2495     c->add_8x8basis = add_8x8basis_c;
2496
2497     c->scalarproduct_and_madd_int16 = scalarproduct_and_madd_int16_c;
2498
2499     c->scalarproduct_int16 = scalarproduct_int16_c;
2500     c->vector_clip_int32   = vector_clip_int32_c;
2501     c->vector_clipf        = vector_clipf_c;
2502
2503     c->shrink[0] = av_image_copy_plane;
2504     c->shrink[1] = ff_shrink22;
2505     c->shrink[2] = ff_shrink44;
2506     c->shrink[3] = ff_shrink88;
2507
2508     c->add_pixels8 = add_pixels8_c;
2509
2510     c->draw_edges = draw_edges_8_c;
2511
2512     c->clear_block  = clear_block_8_c;
2513     c->clear_blocks = clear_blocks_8_c;
2514
2515     switch (avctx->bits_per_raw_sample) {
2516     case 9:
2517     case 10:
2518         c->get_pixels = get_pixels_16_c;
2519         break;
2520     default:
2521         c->get_pixels = get_pixels_8_c;
2522         break;
2523     }
2524
2525     if (ARCH_ARM)
2526         ff_dsputil_init_arm(c, avctx, high_bit_depth);
2527     if (ARCH_BFIN)
2528         ff_dsputil_init_bfin(c, avctx, high_bit_depth);
2529     if (ARCH_PPC)
2530         ff_dsputil_init_ppc(c, avctx, high_bit_depth);
2531     if (ARCH_X86)
2532         ff_dsputil_init_x86(c, avctx, high_bit_depth);
2533
2534     ff_init_scantable_permutation(c->idct_permutation,
2535                                   c->idct_permutation_type);
2536 }