e23f7b149aa30b75bfd15d62b1acf1fed052b7b9
[ffmpeg.git] / libavcodec / arm / h264dsp_neon.S
1 /*
2  * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
3  *
4  * This file is part of Libav.
5  *
6  * Libav is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * Libav is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with Libav; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #include "asm.S"
22 #include "neon.S"
23
24 /* chroma_mc8(uint8_t *dst, uint8_t *src, int stride, int h, int x, int y) */
25 .macro  h264_chroma_mc8 type
26 function ff_\type\()_h264_chroma_mc8_neon, export=1
27         push            {r4-r7, lr}
28         ldrd            r4,  [sp, #20]
29   .ifc \type,avg
30         mov             lr,  r0
31   .endif
32         pld             [r1]
33         pld             [r1, r2]
34
35 A       muls            r7,  r4,  r5
36 T       mul             r7,  r4,  r5
37 T       cmp             r7,  #0
38         rsb             r6,  r7,  r5,  lsl #3
39         rsb             r12, r7,  r4,  lsl #3
40         sub             r4,  r7,  r4,  lsl #3
41         sub             r4,  r4,  r5,  lsl #3
42         add             r4,  r4,  #64
43
44         beq             2f
45
46         add             r5,  r1,  r2
47
48         vdup.8          d0,  r4
49         lsl             r4,  r2,  #1
50         vdup.8          d1,  r12
51         vld1.8          {d4, d5}, [r1], r4
52         vdup.8          d2,  r6
53         vld1.8          {d6, d7}, [r5], r4
54         vdup.8          d3,  r7
55
56         vext.8          d5,  d4,  d5,  #1
57         vext.8          d7,  d6,  d7,  #1
58
59 1:      pld             [r5]
60         vmull.u8        q8,  d4,  d0
61         vmlal.u8        q8,  d5,  d1
62         vld1.8          {d4, d5}, [r1], r4
63         vmlal.u8        q8,  d6,  d2
64         vext.8          d5,  d4,  d5,  #1
65         vmlal.u8        q8,  d7,  d3
66         vmull.u8        q9,  d6,  d0
67         subs            r3,  r3,  #2
68         vmlal.u8        q9,  d7,  d1
69         vmlal.u8        q9,  d4,  d2
70         vmlal.u8        q9,  d5,  d3
71         vrshrn.u16      d16, q8,  #6
72         vld1.8          {d6, d7}, [r5], r4
73         pld             [r1]
74         vrshrn.u16      d17, q9,  #6
75   .ifc \type,avg
76         vld1.8          {d20}, [lr,:64], r2
77         vld1.8          {d21}, [lr,:64], r2
78         vrhadd.u8       q8,  q8,  q10
79   .endif
80         vext.8          d7,  d6,  d7,  #1
81         vst1.8          {d16}, [r0,:64], r2
82         vst1.8          {d17}, [r0,:64], r2
83         bgt             1b
84
85         pop             {r4-r7, pc}
86
87 2:      tst             r6,  r6
88         add             r12, r12, r6
89         vdup.8          d0,  r4
90         vdup.8          d1,  r12
91
92         beq             4f
93
94         add             r5,  r1,  r2
95         lsl             r4,  r2,  #1
96         vld1.8          {d4}, [r1], r4
97         vld1.8          {d6}, [r5], r4
98
99 3:      pld             [r5]
100         vmull.u8        q8,  d4,  d0
101         vmlal.u8        q8,  d6,  d1
102         vld1.8          {d4}, [r1], r4
103         vmull.u8        q9,  d6,  d0
104         vmlal.u8        q9,  d4,  d1
105         vld1.8          {d6}, [r5], r4
106         vrshrn.u16      d16, q8,  #6
107         vrshrn.u16      d17, q9,  #6
108   .ifc \type,avg
109         vld1.8          {d20}, [lr,:64], r2
110         vld1.8          {d21}, [lr,:64], r2
111         vrhadd.u8       q8,  q8,  q10
112   .endif
113         subs            r3,  r3,  #2
114         pld             [r1]
115         vst1.8          {d16}, [r0,:64], r2
116         vst1.8          {d17}, [r0,:64], r2
117         bgt             3b
118
119         pop             {r4-r7, pc}
120
121 4:      vld1.8          {d4, d5}, [r1], r2
122         vld1.8          {d6, d7}, [r1], r2
123         vext.8          d5,  d4,  d5,  #1
124         vext.8          d7,  d6,  d7,  #1
125
126 5:      pld             [r1]
127         subs            r3,  r3,  #2
128         vmull.u8        q8,  d4,  d0
129         vmlal.u8        q8,  d5,  d1
130         vld1.8          {d4, d5}, [r1], r2
131         vmull.u8        q9,  d6,  d0
132         vmlal.u8        q9,  d7,  d1
133         pld             [r1]
134         vext.8          d5,  d4,  d5,  #1
135         vrshrn.u16      d16, q8,  #6
136         vrshrn.u16      d17, q9,  #6
137   .ifc \type,avg
138         vld1.8          {d20}, [lr,:64], r2
139         vld1.8          {d21}, [lr,:64], r2
140         vrhadd.u8       q8,  q8,  q10
141   .endif
142         vld1.8          {d6, d7}, [r1], r2
143         vext.8          d7,  d6,  d7,  #1
144         vst1.8          {d16}, [r0,:64], r2
145         vst1.8          {d17}, [r0,:64], r2
146         bgt             5b
147
148         pop             {r4-r7, pc}
149 endfunc
150 .endm
151
152 /* chroma_mc4(uint8_t *dst, uint8_t *src, int stride, int h, int x, int y) */
153 .macro  h264_chroma_mc4 type
154 function ff_\type\()_h264_chroma_mc4_neon, export=1
155         push            {r4-r7, lr}
156         ldrd            r4,  [sp, #20]
157   .ifc \type,avg
158         mov             lr,  r0
159   .endif
160         pld             [r1]
161         pld             [r1, r2]
162
163 A       muls            r7,  r4,  r5
164 T       mul             r7,  r4,  r5
165 T       cmp             r7,  #0
166         rsb             r6,  r7,  r5,  lsl #3
167         rsb             r12, r7,  r4,  lsl #3
168         sub             r4,  r7,  r4,  lsl #3
169         sub             r4,  r4,  r5,  lsl #3
170         add             r4,  r4,  #64
171
172         beq             2f
173
174         add             r5,  r1,  r2
175
176         vdup.8          d0,  r4
177         lsl             r4,  r2,  #1
178         vdup.8          d1,  r12
179         vld1.8          {d4},     [r1], r4
180         vdup.8          d2,  r6
181         vld1.8          {d6},     [r5], r4
182         vdup.8          d3,  r7
183
184         vext.8          d5,  d4,  d5,  #1
185         vext.8          d7,  d6,  d7,  #1
186         vtrn.32         d4,  d5
187         vtrn.32         d6,  d7
188
189         vtrn.32         d0,  d1
190         vtrn.32         d2,  d3
191
192 1:      pld             [r5]
193         vmull.u8        q8,  d4,  d0
194         vmlal.u8        q8,  d6,  d2
195         vld1.8          {d4},     [r1], r4
196         vext.8          d5,  d4,  d5,  #1
197         vtrn.32         d4,  d5
198         vmull.u8        q9,  d6,  d0
199         vmlal.u8        q9,  d4,  d2
200         vld1.8          {d6},     [r5], r4
201         vadd.i16        d16, d16, d17
202         vadd.i16        d17, d18, d19
203         vrshrn.u16      d16, q8,  #6
204         subs            r3,  r3,  #2
205         pld             [r1]
206   .ifc \type,avg
207         vld1.32         {d20[0]}, [lr,:32], r2
208         vld1.32         {d20[1]}, [lr,:32], r2
209         vrhadd.u8       d16, d16, d20
210   .endif
211         vext.8          d7,  d6,  d7,  #1
212         vtrn.32         d6,  d7
213         vst1.32         {d16[0]}, [r0,:32], r2
214         vst1.32         {d16[1]}, [r0,:32], r2
215         bgt             1b
216
217         pop             {r4-r7, pc}
218
219 2:      tst             r6,  r6
220         add             r12, r12, r6
221         vdup.8          d0,  r4
222         vdup.8          d1,  r12
223         vtrn.32         d0,  d1
224
225         beq             4f
226
227         vext.32         d1,  d0,  d1,  #1
228         add             r5,  r1,  r2
229         lsl             r4,  r2,  #1
230         vld1.32         {d4[0]},  [r1], r4
231         vld1.32         {d4[1]},  [r5], r4
232
233 3:      pld             [r5]
234         vmull.u8        q8,  d4,  d0
235         vld1.32         {d4[0]},  [r1], r4
236         vmull.u8        q9,  d4,  d1
237         vld1.32         {d4[1]},  [r5], r4
238         vadd.i16        d16, d16, d17
239         vadd.i16        d17, d18, d19
240         vrshrn.u16      d16, q8,  #6
241   .ifc \type,avg
242         vld1.32         {d20[0]}, [lr,:32], r2
243         vld1.32         {d20[1]}, [lr,:32], r2
244         vrhadd.u8       d16, d16, d20
245   .endif
246         subs            r3,  r3,  #2
247         pld             [r1]
248         vst1.32         {d16[0]}, [r0,:32], r2
249         vst1.32         {d16[1]}, [r0,:32], r2
250         bgt             3b
251
252         pop             {r4-r7, pc}
253
254 4:      vld1.8          {d4},     [r1], r2
255         vld1.8          {d6},     [r1], r2
256         vext.8          d5,  d4,  d5,  #1
257         vext.8          d7,  d6,  d7,  #1
258         vtrn.32         d4,  d5
259         vtrn.32         d6,  d7
260
261 5:      vmull.u8        q8,  d4,  d0
262         vmull.u8        q9,  d6,  d0
263         subs            r3,  r3,  #2
264         vld1.8          {d4},     [r1], r2
265         vext.8          d5,  d4,  d5,  #1
266         vtrn.32         d4,  d5
267         vadd.i16        d16, d16, d17
268         vadd.i16        d17, d18, d19
269         pld             [r1]
270         vrshrn.u16      d16, q8,  #6
271   .ifc \type,avg
272         vld1.32         {d20[0]}, [lr,:32], r2
273         vld1.32         {d20[1]}, [lr,:32], r2
274         vrhadd.u8       d16, d16, d20
275   .endif
276         vld1.8          {d6},     [r1], r2
277         vext.8          d7,  d6,  d7,  #1
278         vtrn.32         d6,  d7
279         pld             [r1]
280         vst1.32         {d16[0]}, [r0,:32], r2
281         vst1.32         {d16[1]}, [r0,:32], r2
282         bgt             5b
283
284         pop             {r4-r7, pc}
285 endfunc
286 .endm
287
288 .macro  h264_chroma_mc2 type
289 function ff_\type\()_h264_chroma_mc2_neon, export=1
290         push            {r4-r6, lr}
291         ldr             r4,  [sp, #16]
292         ldr             lr,  [sp, #20]
293         pld             [r1]
294         pld             [r1, r2]
295         orrs            r5,  r4,  lr
296         beq             2f
297
298         mul             r5,  r4,  lr
299         rsb             r6,  r5,  lr,  lsl #3
300         rsb             r12, r5,  r4,  lsl #3
301         sub             r4,  r5,  r4,  lsl #3
302         sub             r4,  r4,  lr,  lsl #3
303         add             r4,  r4,  #64
304         vdup.8          d0,  r4
305         vdup.8          d2,  r12
306         vdup.8          d1,  r6
307         vdup.8          d3,  r5
308         vtrn.16         q0,  q1
309 1:
310         vld1.32         {d4[0]},  [r1], r2
311         vld1.32         {d4[1]},  [r1], r2
312         vrev64.32       d5,  d4
313         vld1.32         {d5[1]},  [r1]
314         vext.8          q3,  q2,  q2,  #1
315         vtrn.16         q2,  q3
316         vmull.u8        q8,  d4,  d0
317         vmlal.u8        q8,  d5,  d1
318   .ifc \type,avg
319         vld1.16         {d18[0]}, [r0,:16], r2
320         vld1.16         {d18[1]}, [r0,:16]
321         sub             r0,  r0,  r2
322   .endif
323         vtrn.32         d16, d17
324         vadd.i16        d16, d16, d17
325         vrshrn.u16      d16, q8,  #6
326   .ifc \type,avg
327         vrhadd.u8       d16, d16, d18
328   .endif
329         vst1.16         {d16[0]}, [r0,:16], r2
330         vst1.16         {d16[1]}, [r0,:16], r2
331         subs            r3,  r3,  #2
332         bgt             1b
333         pop             {r4-r6, pc}
334 2:
335   .ifc \type,put
336         ldrh_post       r5,  r1,  r2
337         strh_post       r5,  r0,  r2
338         ldrh_post       r6,  r1,  r2
339         strh_post       r6,  r0,  r2
340   .else
341         vld1.16         {d16[0]}, [r1], r2
342         vld1.16         {d16[1]}, [r1], r2
343         vld1.16         {d18[0]}, [r0,:16], r2
344         vld1.16         {d18[1]}, [r0,:16]
345         sub             r0,  r0,  r2
346         vrhadd.u8       d16, d16, d18
347         vst1.16         {d16[0]}, [r0,:16], r2
348         vst1.16         {d16[1]}, [r0,:16], r2
349   .endif
350         subs            r3,  r3,  #2
351         bgt             2b
352         pop             {r4-r6, pc}
353 endfunc
354 .endm
355
356         h264_chroma_mc8 put
357         h264_chroma_mc8 avg
358         h264_chroma_mc4 put
359         h264_chroma_mc4 avg
360         h264_chroma_mc2 put
361         h264_chroma_mc2 avg
362
363         /* H.264 loop filter */
364
365 .macro  h264_loop_filter_start
366         ldr             r12, [sp]
367         tst             r2,  r2
368         ldr             r12, [r12]
369         it              ne
370         tstne           r3,  r3
371         vmov.32         d24[0], r12
372         and             r12, r12, r12, lsl #16
373         it              eq
374         bxeq            lr
375         ands            r12, r12, r12, lsl #8
376         it              lt
377         bxlt            lr
378 .endm
379
380 .macro  h264_loop_filter_luma
381         vdup.8          q11, r2         @ alpha
382         vmovl.u8        q12, d24
383         vabd.u8         q6,  q8,  q0    @ abs(p0 - q0)
384         vmovl.u16       q12, d24
385         vabd.u8         q14, q9,  q8    @ abs(p1 - p0)
386         vsli.16         q12, q12, #8
387         vabd.u8         q15, q1,  q0    @ abs(q1 - q0)
388         vsli.32         q12, q12, #16
389         vclt.u8         q6,  q6,  q11   @ < alpha
390         vdup.8          q11, r3         @ beta
391         vclt.s8         q7,  q12, #0
392         vclt.u8         q14, q14, q11   @ < beta
393         vclt.u8         q15, q15, q11   @ < beta
394         vbic            q6,  q6,  q7
395         vabd.u8         q4,  q10, q8    @ abs(p2 - p0)
396         vand            q6,  q6,  q14
397         vabd.u8         q5,  q2,  q0    @ abs(q2 - q0)
398         vclt.u8         q4,  q4,  q11   @ < beta
399         vand            q6,  q6,  q15
400         vclt.u8         q5,  q5,  q11   @ < beta
401         vand            q4,  q4,  q6
402         vand            q5,  q5,  q6
403         vand            q12, q12, q6
404         vrhadd.u8       q14, q8,  q0
405         vsub.i8         q6,  q12, q4
406         vqadd.u8        q7,  q9,  q12
407         vhadd.u8        q10, q10, q14
408         vsub.i8         q6,  q6,  q5
409         vhadd.u8        q14, q2,  q14
410         vmin.u8         q7,  q7,  q10
411         vqsub.u8        q11, q9,  q12
412         vqadd.u8        q2,  q1,  q12
413         vmax.u8         q7,  q7,  q11
414         vqsub.u8        q11, q1,  q12
415         vmin.u8         q14, q2,  q14
416         vmovl.u8        q2,  d0
417         vmax.u8         q14, q14, q11
418         vmovl.u8        q10, d1
419         vsubw.u8        q2,  q2,  d16
420         vsubw.u8        q10, q10, d17
421         vshl.i16        q2,  q2,  #2
422         vshl.i16        q10, q10, #2
423         vaddw.u8        q2,  q2,  d18
424         vaddw.u8        q10, q10, d19
425         vsubw.u8        q2,  q2,  d2
426         vsubw.u8        q10, q10, d3
427         vrshrn.i16      d4,  q2,  #3
428         vrshrn.i16      d5,  q10, #3
429         vbsl            q4,  q7,  q9
430         vbsl            q5,  q14, q1
431         vneg.s8         q7,  q6
432         vmovl.u8        q14, d16
433         vmin.s8         q2,  q2,  q6
434         vmovl.u8        q6,  d17
435         vmax.s8         q2,  q2,  q7
436         vmovl.u8        q11, d0
437         vmovl.u8        q12, d1
438         vaddw.s8        q14, q14, d4
439         vaddw.s8        q6,  q6,  d5
440         vsubw.s8        q11, q11, d4
441         vsubw.s8        q12, q12, d5
442         vqmovun.s16     d16, q14
443         vqmovun.s16     d17, q6
444         vqmovun.s16     d0,  q11
445         vqmovun.s16     d1,  q12
446 .endm
447
448 function ff_h264_v_loop_filter_luma_neon, export=1
449         h264_loop_filter_start
450
451         vld1.8          {d0, d1},  [r0,:128], r1
452         vld1.8          {d2, d3},  [r0,:128], r1
453         vld1.8          {d4, d5},  [r0,:128], r1
454         sub             r0,  r0,  r1, lsl #2
455         sub             r0,  r0,  r1, lsl #1
456         vld1.8          {d20,d21}, [r0,:128], r1
457         vld1.8          {d18,d19}, [r0,:128], r1
458         vld1.8          {d16,d17}, [r0,:128], r1
459
460         vpush           {d8-d15}
461
462         h264_loop_filter_luma
463
464         sub             r0,  r0,  r1, lsl #1
465         vst1.8          {d8, d9},  [r0,:128], r1
466         vst1.8          {d16,d17}, [r0,:128], r1
467         vst1.8          {d0, d1},  [r0,:128], r1
468         vst1.8          {d10,d11}, [r0,:128]
469
470         vpop            {d8-d15}
471         bx              lr
472 endfunc
473
474 function ff_h264_h_loop_filter_luma_neon, export=1
475         h264_loop_filter_start
476
477         sub             r0,  r0,  #4
478         vld1.8          {d6},  [r0], r1
479         vld1.8          {d20}, [r0], r1
480         vld1.8          {d18}, [r0], r1
481         vld1.8          {d16}, [r0], r1
482         vld1.8          {d0},  [r0], r1
483         vld1.8          {d2},  [r0], r1
484         vld1.8          {d4},  [r0], r1
485         vld1.8          {d26}, [r0], r1
486         vld1.8          {d7},  [r0], r1
487         vld1.8          {d21}, [r0], r1
488         vld1.8          {d19}, [r0], r1
489         vld1.8          {d17}, [r0], r1
490         vld1.8          {d1},  [r0], r1
491         vld1.8          {d3},  [r0], r1
492         vld1.8          {d5},  [r0], r1
493         vld1.8          {d27}, [r0], r1
494
495         transpose_8x8   q3, q10, q9, q8, q0, q1, q2, q13
496
497         vpush           {d8-d15}
498
499         h264_loop_filter_luma
500
501         transpose_4x4   q4, q8, q0, q5
502
503         sub             r0,  r0,  r1, lsl #4
504         add             r0,  r0,  #2
505         vst1.32         {d8[0]},  [r0], r1
506         vst1.32         {d16[0]}, [r0], r1
507         vst1.32         {d0[0]},  [r0], r1
508         vst1.32         {d10[0]}, [r0], r1
509         vst1.32         {d8[1]},  [r0], r1
510         vst1.32         {d16[1]}, [r0], r1
511         vst1.32         {d0[1]},  [r0], r1
512         vst1.32         {d10[1]}, [r0], r1
513         vst1.32         {d9[0]},  [r0], r1
514         vst1.32         {d17[0]}, [r0], r1
515         vst1.32         {d1[0]},  [r0], r1
516         vst1.32         {d11[0]}, [r0], r1
517         vst1.32         {d9[1]},  [r0], r1
518         vst1.32         {d17[1]}, [r0], r1
519         vst1.32         {d1[1]},  [r0], r1
520         vst1.32         {d11[1]}, [r0], r1
521
522         vpop            {d8-d15}
523         bx              lr
524 endfunc
525
526 .macro  h264_loop_filter_chroma
527         vdup.8          d22, r2         @ alpha
528         vmovl.u8        q12, d24
529         vabd.u8         d26, d16, d0    @ abs(p0 - q0)
530         vmovl.u8        q2,  d0
531         vabd.u8         d28, d18, d16   @ abs(p1 - p0)
532         vsubw.u8        q2,  q2,  d16
533         vsli.16         d24, d24, #8
534         vshl.i16        q2,  q2,  #2
535         vabd.u8         d30, d2,  d0    @ abs(q1 - q0)
536         vaddw.u8        q2,  q2,  d18
537         vclt.u8         d26, d26, d22   @ < alpha
538         vsubw.u8        q2,  q2,  d2
539         vdup.8          d22, r3         @ beta
540         vrshrn.i16      d4,  q2,  #3
541         vclt.u8         d28, d28, d22   @ < beta
542         vclt.u8         d30, d30, d22   @ < beta
543         vmin.s8         d4,  d4,  d24
544         vneg.s8         d25, d24
545         vand            d26, d26, d28
546         vmax.s8         d4,  d4,  d25
547         vand            d26, d26, d30
548         vmovl.u8        q11, d0
549         vand            d4,  d4,  d26
550         vmovl.u8        q14, d16
551         vaddw.s8        q14, q14, d4
552         vsubw.s8        q11, q11, d4
553         vqmovun.s16     d16, q14
554         vqmovun.s16     d0,  q11
555 .endm
556
557 function ff_h264_v_loop_filter_chroma_neon, export=1
558         h264_loop_filter_start
559
560         sub             r0,  r0,  r1, lsl #1
561         vld1.8          {d18}, [r0,:64], r1
562         vld1.8          {d16}, [r0,:64], r1
563         vld1.8          {d0},  [r0,:64], r1
564         vld1.8          {d2},  [r0,:64]
565
566         h264_loop_filter_chroma
567
568         sub             r0,  r0,  r1, lsl #1
569         vst1.8          {d16}, [r0,:64], r1
570         vst1.8          {d0},  [r0,:64], r1
571
572         bx              lr
573 endfunc
574
575 function ff_h264_h_loop_filter_chroma_neon, export=1
576         h264_loop_filter_start
577
578         sub             r0,  r0,  #2
579         vld1.32         {d18[0]}, [r0], r1
580         vld1.32         {d16[0]}, [r0], r1
581         vld1.32         {d0[0]},  [r0], r1
582         vld1.32         {d2[0]},  [r0], r1
583         vld1.32         {d18[1]}, [r0], r1
584         vld1.32         {d16[1]}, [r0], r1
585         vld1.32         {d0[1]},  [r0], r1
586         vld1.32         {d2[1]},  [r0], r1
587
588         vtrn.16         d18, d0
589         vtrn.16         d16, d2
590         vtrn.8          d18, d16
591         vtrn.8          d0,  d2
592
593         h264_loop_filter_chroma
594
595         vtrn.16         d18, d0
596         vtrn.16         d16, d2
597         vtrn.8          d18, d16
598         vtrn.8          d0,  d2
599
600         sub             r0,  r0,  r1, lsl #3
601         vst1.32         {d18[0]}, [r0], r1
602         vst1.32         {d16[0]}, [r0], r1
603         vst1.32         {d0[0]},  [r0], r1
604         vst1.32         {d2[0]},  [r0], r1
605         vst1.32         {d18[1]}, [r0], r1
606         vst1.32         {d16[1]}, [r0], r1
607         vst1.32         {d0[1]},  [r0], r1
608         vst1.32         {d2[1]},  [r0], r1
609
610         bx              lr
611 endfunc
612
613         /* H.264 qpel MC */
614
615 .macro  lowpass_const   r
616         movw            \r,  #5
617         movt            \r,  #20
618         vmov.32         d6[0], \r
619 .endm
620
621 .macro  lowpass_8       r0,  r1,  r2,  r3,  d0,  d1,  narrow=1
622   .if \narrow
623         t0 .req q0
624         t1 .req q8
625   .else
626         t0 .req \d0
627         t1 .req \d1
628   .endif
629         vext.8          d2,  \r0, \r1, #2
630         vext.8          d3,  \r0, \r1, #3
631         vaddl.u8        q1,  d2,  d3
632         vext.8          d4,  \r0, \r1, #1
633         vext.8          d5,  \r0, \r1, #4
634         vaddl.u8        q2,  d4,  d5
635         vext.8          d30, \r0, \r1, #5
636         vaddl.u8        t0,  \r0, d30
637         vext.8          d18, \r2, \r3, #2
638         vmla.i16        t0,  q1,  d6[1]
639         vext.8          d19, \r2, \r3, #3
640         vaddl.u8        q9,  d18, d19
641         vext.8          d20, \r2, \r3, #1
642         vmls.i16        t0,  q2,  d6[0]
643         vext.8          d21, \r2, \r3, #4
644         vaddl.u8        q10, d20, d21
645         vext.8          d31, \r2, \r3, #5
646         vaddl.u8        t1,  \r2, d31
647         vmla.i16        t1,  q9,  d6[1]
648         vmls.i16        t1,  q10, d6[0]
649   .if \narrow
650         vqrshrun.s16    \d0, t0,  #5
651         vqrshrun.s16    \d1, t1,  #5
652   .endif
653         .unreq  t0
654         .unreq  t1
655 .endm
656
657 .macro  lowpass_8_1     r0,  r1,  d0,  narrow=1
658   .if \narrow
659         t0 .req q0
660   .else
661         t0 .req \d0
662   .endif
663         vext.8          d2,  \r0, \r1, #2
664         vext.8          d3,  \r0, \r1, #3
665         vaddl.u8        q1,  d2,  d3
666         vext.8          d4,  \r0, \r1, #1
667         vext.8          d5,  \r0, \r1, #4
668         vaddl.u8        q2,  d4,  d5
669         vext.8          d30, \r0, \r1, #5
670         vaddl.u8        t0,  \r0, d30
671         vmla.i16        t0,  q1,  d6[1]
672         vmls.i16        t0,  q2,  d6[0]
673   .if \narrow
674         vqrshrun.s16    \d0, t0,  #5
675   .endif
676         .unreq  t0
677 .endm
678
679 .macro  lowpass_8.16    r0,  r1,  l0,  h0,  l1,  h1,  d
680         vext.16         q1,  \r0, \r1, #2
681         vext.16         q0,  \r0, \r1, #3
682         vaddl.s16       q9,  d2,  d0
683         vext.16         q2,  \r0, \r1, #1
684         vaddl.s16       q1,  d3,  d1
685         vext.16         q3,  \r0, \r1, #4
686         vaddl.s16       q10, d4,  d6
687         vext.16         \r1, \r0, \r1, #5
688         vaddl.s16       q2,  d5,  d7
689         vaddl.s16       q0,  \h0, \h1
690         vaddl.s16       q8,  \l0, \l1
691
692         vshl.i32        q3,  q9,  #4
693         vshl.i32        q9,  q9,  #2
694         vshl.i32        q15, q10, #2
695         vadd.i32        q9,  q9,  q3
696         vadd.i32        q10, q10, q15
697
698         vshl.i32        q3,  q1,  #4
699         vshl.i32        q1,  q1,  #2
700         vshl.i32        q15, q2,  #2
701         vadd.i32        q1,  q1,  q3
702         vadd.i32        q2,  q2,  q15
703
704         vadd.i32        q9,  q9,  q8
705         vsub.i32        q9,  q9,  q10
706
707         vadd.i32        q1,  q1,  q0
708         vsub.i32        q1,  q1,  q2
709
710         vrshrn.s32      d18, q9,  #10
711         vrshrn.s32      d19, q1,  #10
712
713         vqmovun.s16     \d,  q9
714 .endm
715
716 function put_h264_qpel16_h_lowpass_neon_packed
717         mov             r4,  lr
718         mov             r12, #16
719         mov             r3,  #8
720         bl              put_h264_qpel8_h_lowpass_neon
721         sub             r1,  r1,  r2, lsl #4
722         add             r1,  r1,  #8
723         mov             r12, #16
724         mov             lr,  r4
725         b               put_h264_qpel8_h_lowpass_neon
726 endfunc
727
728 .macro  h264_qpel_h_lowpass type
729 function \type\()_h264_qpel16_h_lowpass_neon
730         push            {lr}
731         mov             r12, #16
732         bl              \type\()_h264_qpel8_h_lowpass_neon
733         sub             r0,  r0,  r3, lsl #4
734         sub             r1,  r1,  r2, lsl #4
735         add             r0,  r0,  #8
736         add             r1,  r1,  #8
737         mov             r12, #16
738         pop             {lr}
739 endfunc
740
741 function \type\()_h264_qpel8_h_lowpass_neon
742 1:      vld1.8          {d0, d1},  [r1], r2
743         vld1.8          {d16,d17}, [r1], r2
744         subs            r12, r12, #2
745         lowpass_8       d0,  d1,  d16, d17, d0,  d16
746   .ifc \type,avg
747         vld1.8          {d2},     [r0,:64], r3
748         vrhadd.u8       d0,  d0,  d2
749         vld1.8          {d3},     [r0,:64]
750         vrhadd.u8       d16, d16, d3
751         sub             r0,  r0,  r3
752   .endif
753         vst1.8          {d0},     [r0,:64], r3
754         vst1.8          {d16},    [r0,:64], r3
755         bne             1b
756         bx              lr
757 endfunc
758 .endm
759
760         h264_qpel_h_lowpass put
761         h264_qpel_h_lowpass avg
762
763 .macro  h264_qpel_h_lowpass_l2 type
764 function \type\()_h264_qpel16_h_lowpass_l2_neon
765         push            {lr}
766         mov             r12, #16
767         bl              \type\()_h264_qpel8_h_lowpass_l2_neon
768         sub             r0,  r0,  r2, lsl #4
769         sub             r1,  r1,  r2, lsl #4
770         sub             r3,  r3,  r2, lsl #4
771         add             r0,  r0,  #8
772         add             r1,  r1,  #8
773         add             r3,  r3,  #8
774         mov             r12, #16
775         pop             {lr}
776 endfunc
777
778 function \type\()_h264_qpel8_h_lowpass_l2_neon
779 1:      vld1.8          {d0, d1},  [r1], r2
780         vld1.8          {d16,d17}, [r1], r2
781         vld1.8          {d28},     [r3], r2
782         vld1.8          {d29},     [r3], r2
783         subs            r12, r12, #2
784         lowpass_8       d0,  d1,  d16, d17, d0,  d1
785         vrhadd.u8       q0,  q0,  q14
786   .ifc \type,avg
787         vld1.8          {d2},      [r0,:64], r2
788         vrhadd.u8       d0,  d0,  d2
789         vld1.8          {d3},      [r0,:64]
790         vrhadd.u8       d1,  d1,  d3
791         sub             r0,  r0,  r2
792   .endif
793         vst1.8          {d0},      [r0,:64], r2
794         vst1.8          {d1},      [r0,:64], r2
795         bne             1b
796         bx              lr
797 endfunc
798 .endm
799
800         h264_qpel_h_lowpass_l2 put
801         h264_qpel_h_lowpass_l2 avg
802
803 function put_h264_qpel16_v_lowpass_neon_packed
804         mov             r4,  lr
805         mov             r2,  #8
806         bl              put_h264_qpel8_v_lowpass_neon
807         sub             r1,  r1,  r3, lsl #2
808         bl              put_h264_qpel8_v_lowpass_neon
809         sub             r1,  r1,  r3, lsl #4
810         sub             r1,  r1,  r3, lsl #2
811         add             r1,  r1,  #8
812         bl              put_h264_qpel8_v_lowpass_neon
813         sub             r1,  r1,  r3, lsl #2
814         mov             lr,  r4
815         b               put_h264_qpel8_v_lowpass_neon
816 endfunc
817
818 .macro  h264_qpel_v_lowpass type
819 function \type\()_h264_qpel16_v_lowpass_neon
820         mov             r4,  lr
821         bl              \type\()_h264_qpel8_v_lowpass_neon
822         sub             r1,  r1,  r3, lsl #2
823         bl              \type\()_h264_qpel8_v_lowpass_neon
824         sub             r0,  r0,  r2, lsl #4
825         add             r0,  r0,  #8
826         sub             r1,  r1,  r3, lsl #4
827         sub             r1,  r1,  r3, lsl #2
828         add             r1,  r1,  #8
829         bl              \type\()_h264_qpel8_v_lowpass_neon
830         sub             r1,  r1,  r3, lsl #2
831         mov             lr,  r4
832 endfunc
833
834 function \type\()_h264_qpel8_v_lowpass_neon
835         vld1.8          {d8},  [r1], r3
836         vld1.8          {d10}, [r1], r3
837         vld1.8          {d12}, [r1], r3
838         vld1.8          {d14}, [r1], r3
839         vld1.8          {d22}, [r1], r3
840         vld1.8          {d24}, [r1], r3
841         vld1.8          {d26}, [r1], r3
842         vld1.8          {d28}, [r1], r3
843         vld1.8          {d9},  [r1], r3
844         vld1.8          {d11}, [r1], r3
845         vld1.8          {d13}, [r1], r3
846         vld1.8          {d15}, [r1], r3
847         vld1.8          {d23}, [r1]
848
849         transpose_8x8   q4,  q5,  q6,  q7,  q11, q12, q13, q14
850         lowpass_8       d8,  d9,  d10, d11, d8,  d10
851         lowpass_8       d12, d13, d14, d15, d12, d14
852         lowpass_8       d22, d23, d24, d25, d22, d24
853         lowpass_8       d26, d27, d28, d29, d26, d28
854         transpose_8x8   d8,  d10, d12, d14, d22, d24, d26, d28
855
856   .ifc \type,avg
857         vld1.8          {d9},  [r0,:64], r2
858         vrhadd.u8       d8,  d8,  d9
859         vld1.8          {d11}, [r0,:64], r2
860         vrhadd.u8       d10, d10, d11
861         vld1.8          {d13}, [r0,:64], r2
862         vrhadd.u8       d12, d12, d13
863         vld1.8          {d15}, [r0,:64], r2
864         vrhadd.u8       d14, d14, d15
865         vld1.8          {d23}, [r0,:64], r2
866         vrhadd.u8       d22, d22, d23
867         vld1.8          {d25}, [r0,:64], r2
868         vrhadd.u8       d24, d24, d25
869         vld1.8          {d27}, [r0,:64], r2
870         vrhadd.u8       d26, d26, d27
871         vld1.8          {d29}, [r0,:64], r2
872         vrhadd.u8       d28, d28, d29
873         sub             r0,  r0,  r2,  lsl #3
874   .endif
875
876         vst1.8          {d8},  [r0,:64], r2
877         vst1.8          {d10}, [r0,:64], r2
878         vst1.8          {d12}, [r0,:64], r2
879         vst1.8          {d14}, [r0,:64], r2
880         vst1.8          {d22}, [r0,:64], r2
881         vst1.8          {d24}, [r0,:64], r2
882         vst1.8          {d26}, [r0,:64], r2
883         vst1.8          {d28}, [r0,:64], r2
884
885         bx              lr
886 endfunc
887 .endm
888
889         h264_qpel_v_lowpass put
890         h264_qpel_v_lowpass avg
891
892 .macro  h264_qpel_v_lowpass_l2 type
893 function \type\()_h264_qpel16_v_lowpass_l2_neon
894         mov             r4,  lr
895         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
896         sub             r1,  r1,  r3, lsl #2
897         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
898         sub             r0,  r0,  r3, lsl #4
899         sub             r12, r12, r2, lsl #4
900         add             r0,  r0,  #8
901         add             r12, r12, #8
902         sub             r1,  r1,  r3, lsl #4
903         sub             r1,  r1,  r3, lsl #2
904         add             r1,  r1,  #8
905         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
906         sub             r1,  r1,  r3, lsl #2
907         mov             lr,  r4
908 endfunc
909
910 function \type\()_h264_qpel8_v_lowpass_l2_neon
911         vld1.8          {d8},  [r1], r3
912         vld1.8          {d10}, [r1], r3
913         vld1.8          {d12}, [r1], r3
914         vld1.8          {d14}, [r1], r3
915         vld1.8          {d22}, [r1], r3
916         vld1.8          {d24}, [r1], r3
917         vld1.8          {d26}, [r1], r3
918         vld1.8          {d28}, [r1], r3
919         vld1.8          {d9},  [r1], r3
920         vld1.8          {d11}, [r1], r3
921         vld1.8          {d13}, [r1], r3
922         vld1.8          {d15}, [r1], r3
923         vld1.8          {d23}, [r1]
924
925         transpose_8x8   q4,  q5,  q6,  q7,  q11, q12, q13, q14
926         lowpass_8       d8,  d9,  d10, d11, d8,  d9
927         lowpass_8       d12, d13, d14, d15, d12, d13
928         lowpass_8       d22, d23, d24, d25, d22, d23
929         lowpass_8       d26, d27, d28, d29, d26, d27
930         transpose_8x8   d8,  d9,  d12, d13, d22, d23, d26, d27
931
932         vld1.8          {d0},  [r12], r2
933         vld1.8          {d1},  [r12], r2
934         vld1.8          {d2},  [r12], r2
935         vld1.8          {d3},  [r12], r2
936         vld1.8          {d4},  [r12], r2
937         vrhadd.u8       q0,  q0,  q4
938         vld1.8          {d5},  [r12], r2
939         vrhadd.u8       q1,  q1,  q6
940         vld1.8          {d10}, [r12], r2
941         vrhadd.u8       q2,  q2,  q11
942         vld1.8          {d11}, [r12], r2
943         vrhadd.u8       q5,  q5,  q13
944
945   .ifc \type,avg
946         vld1.8          {d16}, [r0,:64], r3
947         vrhadd.u8       d0,  d0,  d16
948         vld1.8          {d17}, [r0,:64], r3
949         vrhadd.u8       d1,  d1,  d17
950         vld1.8          {d16}, [r0,:64], r3
951         vrhadd.u8       d2,  d2,  d16
952         vld1.8          {d17}, [r0,:64], r3
953         vrhadd.u8       d3,  d3,  d17
954         vld1.8          {d16}, [r0,:64], r3
955         vrhadd.u8       d4,  d4,  d16
956         vld1.8          {d17}, [r0,:64], r3
957         vrhadd.u8       d5,  d5,  d17
958         vld1.8          {d16}, [r0,:64], r3
959         vrhadd.u8       d10, d10, d16
960         vld1.8          {d17}, [r0,:64], r3
961         vrhadd.u8       d11, d11, d17
962         sub             r0,  r0,  r3,  lsl #3
963   .endif
964
965         vst1.8          {d0},  [r0,:64], r3
966         vst1.8          {d1},  [r0,:64], r3
967         vst1.8          {d2},  [r0,:64], r3
968         vst1.8          {d3},  [r0,:64], r3
969         vst1.8          {d4},  [r0,:64], r3
970         vst1.8          {d5},  [r0,:64], r3
971         vst1.8          {d10}, [r0,:64], r3
972         vst1.8          {d11}, [r0,:64], r3
973
974         bx              lr
975 endfunc
976 .endm
977
978         h264_qpel_v_lowpass_l2 put
979         h264_qpel_v_lowpass_l2 avg
980
981 function put_h264_qpel8_hv_lowpass_neon_top
982         lowpass_const   r12
983         mov             r12, #12
984 1:      vld1.8          {d0, d1},  [r1], r3
985         vld1.8          {d16,d17}, [r1], r3
986         subs            r12, r12, #2
987         lowpass_8       d0,  d1,  d16, d17, q11, q12, narrow=0
988         vst1.8          {d22-d25}, [r4,:128]!
989         bne             1b
990
991         vld1.8          {d0, d1},  [r1]
992         lowpass_8_1     d0,  d1,  q12, narrow=0
993
994         mov             r12, #-16
995         add             r4,  r4,  r12
996         vld1.8          {d30,d31}, [r4,:128], r12
997         vld1.8          {d20,d21}, [r4,:128], r12
998         vld1.8          {d18,d19}, [r4,:128], r12
999         vld1.8          {d16,d17}, [r4,:128], r12
1000         vld1.8          {d14,d15}, [r4,:128], r12
1001         vld1.8          {d12,d13}, [r4,:128], r12
1002         vld1.8          {d10,d11}, [r4,:128], r12
1003         vld1.8          {d8, d9},  [r4,:128], r12
1004         vld1.8          {d6, d7},  [r4,:128], r12
1005         vld1.8          {d4, d5},  [r4,:128], r12
1006         vld1.8          {d2, d3},  [r4,:128], r12
1007         vld1.8          {d0, d1},  [r4,:128]
1008
1009         swap4           d1,  d3,  d5,  d7,  d8,  d10, d12, d14
1010         transpose16_4x4 q0,  q1,  q2,  q3,  q4,  q5,  q6,  q7
1011
1012         swap4           d17, d19, d21, d31, d24, d26, d28, d22
1013         transpose16_4x4 q8,  q9,  q10, q15, q12, q13, q14, q11
1014
1015         vst1.8          {d30,d31}, [r4,:128]!
1016         vst1.8          {d6, d7},  [r4,:128]!
1017         vst1.8          {d20,d21}, [r4,:128]!
1018         vst1.8          {d4, d5},  [r4,:128]!
1019         vst1.8          {d18,d19}, [r4,:128]!
1020         vst1.8          {d2, d3},  [r4,:128]!
1021         vst1.8          {d16,d17}, [r4,:128]!
1022         vst1.8          {d0, d1},  [r4,:128]
1023
1024         lowpass_8.16    q4,  q12, d8,  d9,  d24, d25, d8
1025         lowpass_8.16    q5,  q13, d10, d11, d26, d27, d9
1026         lowpass_8.16    q6,  q14, d12, d13, d28, d29, d10
1027         lowpass_8.16    q7,  q11, d14, d15, d22, d23, d11
1028
1029         vld1.8          {d16,d17}, [r4,:128], r12
1030         vld1.8          {d30,d31}, [r4,:128], r12
1031         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d12
1032         vld1.8          {d16,d17}, [r4,:128], r12
1033         vld1.8          {d30,d31}, [r4,:128], r12
1034         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d13
1035         vld1.8          {d16,d17}, [r4,:128], r12
1036         vld1.8          {d30,d31}, [r4,:128], r12
1037         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d14
1038         vld1.8          {d16,d17}, [r4,:128], r12
1039         vld1.8          {d30,d31}, [r4,:128]
1040         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d15
1041
1042         transpose_8x8   d12, d13, d14, d15, d8,  d9,  d10, d11
1043
1044         bx              lr
1045 endfunc
1046
1047 .macro  h264_qpel8_hv_lowpass type
1048 function \type\()_h264_qpel8_hv_lowpass_neon
1049         mov             r10, lr
1050         bl              put_h264_qpel8_hv_lowpass_neon_top
1051   .ifc \type,avg
1052         vld1.8          {d0},      [r0,:64], r2
1053         vrhadd.u8       d12, d12, d0
1054         vld1.8          {d1},      [r0,:64], r2
1055         vrhadd.u8       d13, d13, d1
1056         vld1.8          {d2},      [r0,:64], r2
1057         vrhadd.u8       d14, d14, d2
1058         vld1.8          {d3},      [r0,:64], r2
1059         vrhadd.u8       d15, d15, d3
1060         vld1.8          {d4},      [r0,:64], r2
1061         vrhadd.u8       d8,  d8,  d4
1062         vld1.8          {d5},      [r0,:64], r2
1063         vrhadd.u8       d9,  d9,  d5
1064         vld1.8          {d6},      [r0,:64], r2
1065         vrhadd.u8       d10, d10, d6
1066         vld1.8          {d7},      [r0,:64], r2
1067         vrhadd.u8       d11, d11, d7
1068         sub             r0,  r0,  r2,  lsl #3
1069   .endif
1070
1071         vst1.8          {d12},     [r0,:64], r2
1072         vst1.8          {d13},     [r0,:64], r2
1073         vst1.8          {d14},     [r0,:64], r2
1074         vst1.8          {d15},     [r0,:64], r2
1075         vst1.8          {d8},      [r0,:64], r2
1076         vst1.8          {d9},      [r0,:64], r2
1077         vst1.8          {d10},     [r0,:64], r2
1078         vst1.8          {d11},     [r0,:64], r2
1079
1080         mov             lr,  r10
1081         bx              lr
1082 endfunc
1083 .endm
1084
1085         h264_qpel8_hv_lowpass put
1086         h264_qpel8_hv_lowpass avg
1087
1088 .macro  h264_qpel8_hv_lowpass_l2 type
1089 function \type\()_h264_qpel8_hv_lowpass_l2_neon
1090         mov             r10, lr
1091         bl              put_h264_qpel8_hv_lowpass_neon_top
1092
1093         vld1.8          {d0, d1},  [r2,:128]!
1094         vld1.8          {d2, d3},  [r2,:128]!
1095         vrhadd.u8       q0,  q0,  q6
1096         vld1.8          {d4, d5},  [r2,:128]!
1097         vrhadd.u8       q1,  q1,  q7
1098         vld1.8          {d6, d7},  [r2,:128]!
1099         vrhadd.u8       q2,  q2,  q4
1100         vrhadd.u8       q3,  q3,  q5
1101   .ifc \type,avg
1102         vld1.8          {d16},     [r0,:64], r3
1103         vrhadd.u8       d0,  d0,  d16
1104         vld1.8          {d17},     [r0,:64], r3
1105         vrhadd.u8       d1,  d1,  d17
1106         vld1.8          {d18},     [r0,:64], r3
1107         vrhadd.u8       d2,  d2,  d18
1108         vld1.8          {d19},     [r0,:64], r3
1109         vrhadd.u8       d3,  d3,  d19
1110         vld1.8          {d20},     [r0,:64], r3
1111         vrhadd.u8       d4,  d4,  d20
1112         vld1.8          {d21},     [r0,:64], r3
1113         vrhadd.u8       d5,  d5,  d21
1114         vld1.8          {d22},     [r0,:64], r3
1115         vrhadd.u8       d6,  d6,  d22
1116         vld1.8          {d23},     [r0,:64], r3
1117         vrhadd.u8       d7,  d7,  d23
1118         sub             r0,  r0,  r3,  lsl #3
1119   .endif
1120         vst1.8          {d0},      [r0,:64], r3
1121         vst1.8          {d1},      [r0,:64], r3
1122         vst1.8          {d2},      [r0,:64], r3
1123         vst1.8          {d3},      [r0,:64], r3
1124         vst1.8          {d4},      [r0,:64], r3
1125         vst1.8          {d5},      [r0,:64], r3
1126         vst1.8          {d6},      [r0,:64], r3
1127         vst1.8          {d7},      [r0,:64], r3
1128
1129         mov             lr,  r10
1130         bx              lr
1131 endfunc
1132 .endm
1133
1134         h264_qpel8_hv_lowpass_l2 put
1135         h264_qpel8_hv_lowpass_l2 avg
1136
1137 .macro  h264_qpel16_hv  type
1138 function \type\()_h264_qpel16_hv_lowpass_neon
1139         mov             r9,  lr
1140         bl              \type\()_h264_qpel8_hv_lowpass_neon
1141         sub             r1,  r1,  r3, lsl #2
1142         bl              \type\()_h264_qpel8_hv_lowpass_neon
1143         sub             r1,  r1,  r3, lsl #4
1144         sub             r1,  r1,  r3, lsl #2
1145         add             r1,  r1,  #8
1146         sub             r0,  r0,  r2, lsl #4
1147         add             r0,  r0,  #8
1148         bl              \type\()_h264_qpel8_hv_lowpass_neon
1149         sub             r1,  r1,  r3, lsl #2
1150         mov             lr,  r9
1151         b               \type\()_h264_qpel8_hv_lowpass_neon
1152 endfunc
1153
1154 function \type\()_h264_qpel16_hv_lowpass_l2_neon
1155         mov             r9,  lr
1156         sub             r2,  r4,  #256
1157         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
1158         sub             r1,  r1,  r3, lsl #2
1159         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
1160         sub             r1,  r1,  r3, lsl #4
1161         sub             r1,  r1,  r3, lsl #2
1162         add             r1,  r1,  #8
1163         sub             r0,  r0,  r3, lsl #4
1164         add             r0,  r0,  #8
1165         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
1166         sub             r1,  r1,  r3, lsl #2
1167         mov             lr,  r9
1168         b               \type\()_h264_qpel8_hv_lowpass_l2_neon
1169 endfunc
1170 .endm
1171
1172         h264_qpel16_hv put
1173         h264_qpel16_hv avg
1174
1175 .macro  h264_qpel8      type
1176 function ff_\type\()_h264_qpel8_mc10_neon, export=1
1177         lowpass_const   r3
1178         mov             r3,  r1
1179         sub             r1,  r1,  #2
1180         mov             r12, #8
1181         b               \type\()_h264_qpel8_h_lowpass_l2_neon
1182 endfunc
1183
1184 function ff_\type\()_h264_qpel8_mc20_neon, export=1
1185         lowpass_const   r3
1186         sub             r1,  r1,  #2
1187         mov             r3,  r2
1188         mov             r12, #8
1189         b               \type\()_h264_qpel8_h_lowpass_neon
1190 endfunc
1191
1192 function ff_\type\()_h264_qpel8_mc30_neon, export=1
1193         lowpass_const   r3
1194         add             r3,  r1,  #1
1195         sub             r1,  r1,  #2
1196         mov             r12, #8
1197         b               \type\()_h264_qpel8_h_lowpass_l2_neon
1198 endfunc
1199
1200 function ff_\type\()_h264_qpel8_mc01_neon, export=1
1201         push            {lr}
1202         mov             r12, r1
1203 \type\()_h264_qpel8_mc01:
1204         lowpass_const   r3
1205         mov             r3,  r2
1206         sub             r1,  r1,  r2, lsl #1
1207         vpush           {d8-d15}
1208         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
1209         vpop            {d8-d15}
1210         pop             {pc}
1211 endfunc
1212
1213 function ff_\type\()_h264_qpel8_mc11_neon, export=1
1214         push            {r0, r1, r11, lr}
1215 \type\()_h264_qpel8_mc11:
1216         lowpass_const   r3
1217         mov             r11, sp
1218 A       bic             sp,  sp,  #15
1219 T       bic             r0,  r11, #15
1220 T       mov             sp,  r0
1221         sub             sp,  sp,  #64
1222         mov             r0,  sp
1223         sub             r1,  r1,  #2
1224         mov             r3,  #8
1225         mov             r12, #8
1226         vpush           {d8-d15}
1227         bl              put_h264_qpel8_h_lowpass_neon
1228         ldrd            r0,  [r11], #8
1229         mov             r3,  r2
1230         add             r12, sp,  #64
1231         sub             r1,  r1,  r2, lsl #1
1232         mov             r2,  #8
1233         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
1234         vpop            {d8-d15}
1235         mov             sp,  r11
1236         pop             {r11, pc}
1237 endfunc
1238
1239 function ff_\type\()_h264_qpel8_mc21_neon, export=1
1240         push            {r0, r1, r4, r10, r11, lr}
1241 \type\()_h264_qpel8_mc21:
1242         lowpass_const   r3
1243         mov             r11, sp
1244 A       bic             sp,  sp,  #15
1245 T       bic             r0,  r11, #15
1246 T       mov             sp,  r0
1247         sub             sp,  sp,  #(8*8+16*12)
1248         sub             r1,  r1,  #2
1249         mov             r3,  #8
1250         mov             r0,  sp
1251         mov             r12, #8
1252         vpush           {d8-d15}
1253         bl              put_h264_qpel8_h_lowpass_neon
1254         mov             r4,  r0
1255         ldrd            r0,  [r11], #8
1256         sub             r1,  r1,  r2, lsl #1
1257         sub             r1,  r1,  #2
1258         mov             r3,  r2
1259         sub             r2,  r4,  #64
1260         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
1261         vpop            {d8-d15}
1262         mov             sp,  r11
1263         pop             {r4, r10, r11, pc}
1264 endfunc
1265
1266 function ff_\type\()_h264_qpel8_mc31_neon, export=1
1267         add             r1,  r1,  #1
1268         push            {r0, r1, r11, lr}
1269         sub             r1,  r1,  #1
1270         b               \type\()_h264_qpel8_mc11
1271 endfunc
1272
1273 function ff_\type\()_h264_qpel8_mc02_neon, export=1
1274         push            {lr}
1275         lowpass_const   r3
1276         sub             r1,  r1,  r2, lsl #1
1277         mov             r3,  r2
1278         vpush           {d8-d15}
1279         bl              \type\()_h264_qpel8_v_lowpass_neon
1280         vpop            {d8-d15}
1281         pop             {pc}
1282 endfunc
1283
1284 function ff_\type\()_h264_qpel8_mc12_neon, export=1
1285         push            {r0, r1, r4, r10, r11, lr}
1286 \type\()_h264_qpel8_mc12:
1287         lowpass_const   r3
1288         mov             r11, sp
1289 A       bic             sp,  sp,  #15
1290 T       bic             r0,  r11, #15
1291 T       mov             sp,  r0
1292         sub             sp,  sp,  #(8*8+16*12)
1293         sub             r1,  r1,  r2, lsl #1
1294         mov             r3,  r2
1295         mov             r2,  #8
1296         mov             r0,  sp
1297         vpush           {d8-d15}
1298         bl              put_h264_qpel8_v_lowpass_neon
1299         mov             r4,  r0
1300         ldrd            r0,  [r11], #8
1301         sub             r1,  r1,  r3, lsl #1
1302         sub             r1,  r1,  #2
1303         sub             r2,  r4,  #64
1304         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
1305         vpop            {d8-d15}
1306         mov             sp,  r11
1307         pop             {r4, r10, r11, pc}
1308 endfunc
1309
1310 function ff_\type\()_h264_qpel8_mc22_neon, export=1
1311         push            {r4, r10, r11, lr}
1312         mov             r11, sp
1313 A       bic             sp,  sp,  #15
1314 T       bic             r4,  r11, #15
1315 T       mov             sp,  r4
1316         sub             r1,  r1,  r2, lsl #1
1317         sub             r1,  r1,  #2
1318         mov             r3,  r2
1319         sub             sp,  sp,  #(16*12)
1320         mov             r4,  sp
1321         vpush           {d8-d15}
1322         bl              \type\()_h264_qpel8_hv_lowpass_neon
1323         vpop            {d8-d15}
1324         mov             sp,  r11
1325         pop             {r4, r10, r11, pc}
1326 endfunc
1327
1328 function ff_\type\()_h264_qpel8_mc32_neon, export=1
1329         push            {r0, r1, r4, r10, r11, lr}
1330         add             r1,  r1,  #1
1331         b               \type\()_h264_qpel8_mc12
1332 endfunc
1333
1334 function ff_\type\()_h264_qpel8_mc03_neon, export=1
1335         push            {lr}
1336         add             r12, r1,  r2
1337         b               \type\()_h264_qpel8_mc01
1338 endfunc
1339
1340 function ff_\type\()_h264_qpel8_mc13_neon, export=1
1341         push            {r0, r1, r11, lr}
1342         add             r1,  r1,  r2
1343         b               \type\()_h264_qpel8_mc11
1344 endfunc
1345
1346 function ff_\type\()_h264_qpel8_mc23_neon, export=1
1347         push            {r0, r1, r4, r10, r11, lr}
1348         add             r1,  r1,  r2
1349         b               \type\()_h264_qpel8_mc21
1350 endfunc
1351
1352 function ff_\type\()_h264_qpel8_mc33_neon, export=1
1353         add             r1,  r1,  #1
1354         push            {r0, r1, r11, lr}
1355         add             r1,  r1,  r2
1356         sub             r1,  r1,  #1
1357         b               \type\()_h264_qpel8_mc11
1358 endfunc
1359 .endm
1360
1361         h264_qpel8 put
1362         h264_qpel8 avg
1363
1364 .macro  h264_qpel16     type
1365 function ff_\type\()_h264_qpel16_mc10_neon, export=1
1366         lowpass_const   r3
1367         mov             r3,  r1
1368         sub             r1,  r1,  #2
1369         b               \type\()_h264_qpel16_h_lowpass_l2_neon
1370 endfunc
1371
1372 function ff_\type\()_h264_qpel16_mc20_neon, export=1
1373         lowpass_const   r3
1374         sub             r1,  r1,  #2
1375         mov             r3,  r2
1376         b               \type\()_h264_qpel16_h_lowpass_neon
1377 endfunc
1378
1379 function ff_\type\()_h264_qpel16_mc30_neon, export=1
1380         lowpass_const   r3
1381         add             r3,  r1,  #1
1382         sub             r1,  r1,  #2
1383         b               \type\()_h264_qpel16_h_lowpass_l2_neon
1384 endfunc
1385
1386 function ff_\type\()_h264_qpel16_mc01_neon, export=1
1387         push            {r4, lr}
1388         mov             r12, r1
1389 \type\()_h264_qpel16_mc01:
1390         lowpass_const   r3
1391         mov             r3,  r2
1392         sub             r1,  r1,  r2, lsl #1
1393         vpush           {d8-d15}
1394         bl              \type\()_h264_qpel16_v_lowpass_l2_neon
1395         vpop            {d8-d15}
1396         pop             {r4, pc}
1397 endfunc
1398
1399 function ff_\type\()_h264_qpel16_mc11_neon, export=1
1400         push            {r0, r1, r4, r11, lr}
1401 \type\()_h264_qpel16_mc11:
1402         lowpass_const   r3
1403         mov             r11, sp
1404 A       bic             sp,  sp,  #15
1405 T       bic             r0,  r11, #15
1406 T       mov             sp,  r0
1407         sub             sp,  sp,  #256
1408         mov             r0,  sp
1409         sub             r1,  r1,  #2
1410         mov             r3,  #16
1411         vpush           {d8-d15}
1412         bl              put_h264_qpel16_h_lowpass_neon
1413         ldrd            r0,  [r11], #8
1414         mov             r3,  r2
1415         add             r12, sp,  #64
1416         sub             r1,  r1,  r2, lsl #1
1417         mov             r2,  #16
1418         bl              \type\()_h264_qpel16_v_lowpass_l2_neon
1419         vpop            {d8-d15}
1420         mov             sp,  r11
1421         pop             {r4, r11, pc}
1422 endfunc
1423
1424 function ff_\type\()_h264_qpel16_mc21_neon, export=1
1425         push            {r0, r1, r4-r5, r9-r11, lr}
1426 \type\()_h264_qpel16_mc21:
1427         lowpass_const   r3
1428         mov             r11, sp
1429 A       bic             sp,  sp,  #15
1430 T       bic             r0,  r11, #15
1431 T       mov             sp,  r0
1432         sub             sp,  sp,  #(16*16+16*12)
1433         sub             r1,  r1,  #2
1434         mov             r0,  sp
1435         vpush           {d8-d15}
1436         bl              put_h264_qpel16_h_lowpass_neon_packed
1437         mov             r4,  r0
1438         ldrd            r0,  [r11], #8
1439         sub             r1,  r1,  r2, lsl #1
1440         sub             r1,  r1,  #2
1441         mov             r3,  r2
1442         bl              \type\()_h264_qpel16_hv_lowpass_l2_neon
1443         vpop            {d8-d15}
1444         mov             sp,  r11
1445         pop             {r4-r5, r9-r11, pc}
1446 endfunc
1447
1448 function ff_\type\()_h264_qpel16_mc31_neon, export=1
1449         add             r1,  r1,  #1
1450         push            {r0, r1, r4, r11, lr}
1451         sub             r1,  r1,  #1
1452         b               \type\()_h264_qpel16_mc11
1453 endfunc
1454
1455 function ff_\type\()_h264_qpel16_mc02_neon, export=1
1456         push            {r4, lr}
1457         lowpass_const   r3
1458         sub             r1,  r1,  r2, lsl #1
1459         mov             r3,  r2
1460         vpush           {d8-d15}
1461         bl              \type\()_h264_qpel16_v_lowpass_neon
1462         vpop            {d8-d15}
1463         pop             {r4, pc}
1464 endfunc
1465
1466 function ff_\type\()_h264_qpel16_mc12_neon, export=1
1467         push            {r0, r1, r4-r5, r9-r11, lr}
1468 \type\()_h264_qpel16_mc12:
1469         lowpass_const   r3
1470         mov             r11, sp
1471 A       bic             sp,  sp,  #15
1472 T       bic             r0,  r11, #15
1473 T       mov             sp,  r0
1474         sub             sp,  sp,  #(16*16+16*12)
1475         sub             r1,  r1,  r2, lsl #1
1476         mov             r0,  sp
1477         mov             r3,  r2
1478         vpush           {d8-d15}
1479         bl              put_h264_qpel16_v_lowpass_neon_packed
1480         mov             r4,  r0
1481         ldrd            r0,  [r11], #8
1482         sub             r1,  r1,  r3, lsl #1
1483         sub             r1,  r1,  #2
1484         mov             r2,  r3
1485         bl              \type\()_h264_qpel16_hv_lowpass_l2_neon
1486         vpop            {d8-d15}
1487         mov             sp,  r11
1488         pop             {r4-r5, r9-r11, pc}
1489 endfunc
1490
1491 function ff_\type\()_h264_qpel16_mc22_neon, export=1
1492         push            {r4, r9-r11, lr}
1493         lowpass_const   r3
1494         mov             r11, sp
1495 A       bic             sp,  sp,  #15
1496 T       bic             r4,  r11, #15
1497 T       mov             sp,  r4
1498         sub             r1,  r1,  r2, lsl #1
1499         sub             r1,  r1,  #2
1500         mov             r3,  r2
1501         sub             sp,  sp,  #(16*12)
1502         mov             r4,  sp
1503         vpush           {d8-d15}
1504         bl              \type\()_h264_qpel16_hv_lowpass_neon
1505         vpop            {d8-d15}
1506         mov             sp,  r11
1507         pop             {r4, r9-r11, pc}
1508 endfunc
1509
1510 function ff_\type\()_h264_qpel16_mc32_neon, export=1
1511         push            {r0, r1, r4-r5, r9-r11, lr}
1512         add             r1,  r1,  #1
1513         b               \type\()_h264_qpel16_mc12
1514 endfunc
1515
1516 function ff_\type\()_h264_qpel16_mc03_neon, export=1
1517         push            {r4, lr}
1518         add             r12, r1,  r2
1519         b               \type\()_h264_qpel16_mc01
1520 endfunc
1521
1522 function ff_\type\()_h264_qpel16_mc13_neon, export=1
1523         push            {r0, r1, r4, r11, lr}
1524         add             r1,  r1,  r2
1525         b               \type\()_h264_qpel16_mc11
1526 endfunc
1527
1528 function ff_\type\()_h264_qpel16_mc23_neon, export=1
1529         push            {r0, r1, r4-r5, r9-r11, lr}
1530         add             r1,  r1,  r2
1531         b               \type\()_h264_qpel16_mc21
1532 endfunc
1533
1534 function ff_\type\()_h264_qpel16_mc33_neon, export=1
1535         add             r1,  r1,  #1
1536         push            {r0, r1, r4, r11, lr}
1537         add             r1,  r1,  r2
1538         sub             r1,  r1,  #1
1539         b               \type\()_h264_qpel16_mc11
1540 endfunc
1541 .endm
1542
1543         h264_qpel16 put
1544         h264_qpel16 avg
1545
1546 @ Biweighted prediction
1547
1548 .macro  biweight_16     macs, macd
1549         vdup.8          d0,  r4
1550         vdup.8          d1,  r5
1551         vmov            q2,  q8
1552         vmov            q3,  q8
1553 1:      subs            r3,  r3,  #2
1554         vld1.8          {d20-d21},[r0,:128], r2
1555         \macd           q2,  d0,  d20
1556         pld             [r0]
1557         \macd           q3,  d0,  d21
1558         vld1.8          {d22-d23},[r1,:128], r2
1559         \macs           q2,  d1,  d22
1560         pld             [r1]
1561         \macs           q3,  d1,  d23
1562         vmov            q12, q8
1563         vld1.8          {d28-d29},[r0,:128], r2
1564         vmov            q13, q8
1565         \macd           q12, d0,  d28
1566         pld             [r0]
1567         \macd           q13, d0,  d29
1568         vld1.8          {d30-d31},[r1,:128], r2
1569         \macs           q12, d1,  d30
1570         pld             [r1]
1571         \macs           q13, d1,  d31
1572         vshl.s16        q2,  q2,  q9
1573         vshl.s16        q3,  q3,  q9
1574         vqmovun.s16     d4,  q2
1575         vqmovun.s16     d5,  q3
1576         vshl.s16        q12, q12, q9
1577         vshl.s16        q13, q13, q9
1578         vqmovun.s16     d24, q12
1579         vqmovun.s16     d25, q13
1580         vmov            q3,  q8
1581         vst1.8          {d4- d5}, [r6,:128], r2
1582         vmov            q2,  q8
1583         vst1.8          {d24-d25},[r6,:128], r2
1584         bne             1b
1585         pop             {r4-r6, pc}
1586 .endm
1587
1588 .macro  biweight_8      macs, macd
1589         vdup.8          d0,  r4
1590         vdup.8          d1,  r5
1591         vmov            q1,  q8
1592         vmov            q10, q8
1593 1:      subs            r3,  r3,  #2
1594         vld1.8          {d4},[r0,:64], r2
1595         \macd           q1,  d0,  d4
1596         pld             [r0]
1597         vld1.8          {d5},[r1,:64], r2
1598         \macs           q1,  d1,  d5
1599         pld             [r1]
1600         vld1.8          {d6},[r0,:64], r2
1601         \macd           q10, d0,  d6
1602         pld             [r0]
1603         vld1.8          {d7},[r1,:64], r2
1604         \macs           q10, d1,  d7
1605         pld             [r1]
1606         vshl.s16        q1,  q1,  q9
1607         vqmovun.s16     d2,  q1
1608         vshl.s16        q10, q10, q9
1609         vqmovun.s16     d4,  q10
1610         vmov            q10, q8
1611         vst1.8          {d2},[r6,:64], r2
1612         vmov            q1,  q8
1613         vst1.8          {d4},[r6,:64], r2
1614         bne             1b
1615         pop             {r4-r6, pc}
1616 .endm
1617
1618 .macro  biweight_4      macs, macd
1619         vdup.8          d0,  r4
1620         vdup.8          d1,  r5
1621         vmov            q1,  q8
1622         vmov            q10, q8
1623 1:      subs            r3,  r3,  #4
1624         vld1.32         {d4[0]},[r0,:32], r2
1625         vld1.32         {d4[1]},[r0,:32], r2
1626         \macd           q1,  d0,  d4
1627         pld             [r0]
1628         vld1.32         {d5[0]},[r1,:32], r2
1629         vld1.32         {d5[1]},[r1,:32], r2
1630         \macs           q1,  d1,  d5
1631         pld             [r1]
1632         blt             2f
1633         vld1.32         {d6[0]},[r0,:32], r2
1634         vld1.32         {d6[1]},[r0,:32], r2
1635         \macd           q10, d0,  d6
1636         pld             [r0]
1637         vld1.32         {d7[0]},[r1,:32], r2
1638         vld1.32         {d7[1]},[r1,:32], r2
1639         \macs           q10, d1,  d7
1640         pld             [r1]
1641         vshl.s16        q1,  q1,  q9
1642         vqmovun.s16     d2,  q1
1643         vshl.s16        q10, q10, q9
1644         vqmovun.s16     d4,  q10
1645         vmov            q10, q8
1646         vst1.32         {d2[0]},[r6,:32], r2
1647         vst1.32         {d2[1]},[r6,:32], r2
1648         vmov            q1,  q8
1649         vst1.32         {d4[0]},[r6,:32], r2
1650         vst1.32         {d4[1]},[r6,:32], r2
1651         bne             1b
1652         pop             {r4-r6, pc}
1653 2:      vshl.s16        q1,  q1,  q9
1654         vqmovun.s16     d2,  q1
1655         vst1.32         {d2[0]},[r6,:32], r2
1656         vst1.32         {d2[1]},[r6,:32], r2
1657         pop             {r4-r6, pc}
1658 .endm
1659
1660 .macro  biweight_func   w
1661 function ff_biweight_h264_pixels_\w\()_neon, export=1
1662         push            {r4-r6, lr}
1663         ldr             r12, [sp, #16]
1664         add             r4,  sp,  #20
1665         ldm             r4,  {r4-r6}
1666         lsr             lr,  r4,  #31
1667         add             r6,  r6,  #1
1668         eors            lr,  lr,  r5,  lsr #30
1669         orr             r6,  r6,  #1
1670         vdup.16         q9,  r12
1671         lsl             r6,  r6,  r12
1672         vmvn            q9,  q9
1673         vdup.16         q8,  r6
1674         mov             r6,  r0
1675         beq             10f
1676         subs            lr,  lr,  #1
1677         beq             20f
1678         subs            lr,  lr,  #1
1679         beq             30f
1680         b               40f
1681 10:     biweight_\w     vmlal.u8, vmlal.u8
1682 20:     rsb             r4,  r4,  #0
1683         biweight_\w     vmlal.u8, vmlsl.u8
1684 30:     rsb             r4,  r4,  #0
1685         rsb             r5,  r5,  #0
1686         biweight_\w     vmlsl.u8, vmlsl.u8
1687 40:     rsb             r5,  r5,  #0
1688         biweight_\w     vmlsl.u8, vmlal.u8
1689 endfunc
1690 .endm
1691
1692         biweight_func   16
1693         biweight_func   8
1694         biweight_func   4
1695
1696 @ Weighted prediction
1697
1698 .macro  weight_16       add
1699         vdup.8          d0,  r12
1700 1:      subs            r2,  r2,  #2
1701         vld1.8          {d20-d21},[r0,:128], r1
1702         vmull.u8        q2,  d0,  d20
1703         pld             [r0]
1704         vmull.u8        q3,  d0,  d21
1705         vld1.8          {d28-d29},[r0,:128], r1
1706         vmull.u8        q12, d0,  d28
1707         pld             [r0]
1708         vmull.u8        q13, d0,  d29
1709         \add            q2,  q8,  q2
1710         vrshl.s16       q2,  q2,  q9
1711         \add            q3,  q8,  q3
1712         vrshl.s16       q3,  q3,  q9
1713         vqmovun.s16     d4,  q2
1714         vqmovun.s16     d5,  q3
1715         \add            q12, q8,  q12
1716         vrshl.s16       q12, q12, q9
1717         \add            q13, q8,  q13
1718         vrshl.s16       q13, q13, q9
1719         vqmovun.s16     d24, q12
1720         vqmovun.s16     d25, q13
1721         vst1.8          {d4- d5}, [r4,:128], r1
1722         vst1.8          {d24-d25},[r4,:128], r1
1723         bne             1b
1724         pop             {r4, pc}
1725 .endm
1726
1727 .macro  weight_8        add
1728         vdup.8          d0,  r12
1729 1:      subs            r2,  r2,  #2
1730         vld1.8          {d4},[r0,:64], r1
1731         vmull.u8        q1,  d0,  d4
1732         pld             [r0]
1733         vld1.8          {d6},[r0,:64], r1
1734         vmull.u8        q10, d0,  d6
1735         \add            q1,  q8,  q1
1736         pld             [r0]
1737         vrshl.s16       q1,  q1,  q9
1738         vqmovun.s16     d2,  q1
1739         \add            q10, q8,  q10
1740         vrshl.s16       q10, q10, q9
1741         vqmovun.s16     d4,  q10
1742         vst1.8          {d2},[r4,:64], r1
1743         vst1.8          {d4},[r4,:64], r1
1744         bne             1b
1745         pop             {r4, pc}
1746 .endm
1747
1748 .macro  weight_4        add
1749         vdup.8          d0,  r12
1750         vmov            q1,  q8
1751         vmov            q10, q8
1752 1:      subs            r2,  r2,  #4
1753         vld1.32         {d4[0]},[r0,:32], r1
1754         vld1.32         {d4[1]},[r0,:32], r1
1755         vmull.u8        q1,  d0,  d4
1756         pld             [r0]
1757         blt             2f
1758         vld1.32         {d6[0]},[r0,:32], r1
1759         vld1.32         {d6[1]},[r0,:32], r1
1760         vmull.u8        q10, d0,  d6
1761         pld             [r0]
1762         \add            q1,  q8,  q1
1763         vrshl.s16       q1,  q1,  q9
1764         vqmovun.s16     d2,  q1
1765         \add            q10, q8,  q10
1766         vrshl.s16       q10, q10, q9
1767         vqmovun.s16     d4,  q10
1768         vmov            q10, q8
1769         vst1.32         {d2[0]},[r4,:32], r1
1770         vst1.32         {d2[1]},[r4,:32], r1
1771         vmov            q1,  q8
1772         vst1.32         {d4[0]},[r4,:32], r1
1773         vst1.32         {d4[1]},[r4,:32], r1
1774         bne             1b
1775         pop             {r4, pc}
1776 2:      \add            q1,  q8,  q1
1777         vrshl.s16       q1,  q1,  q9
1778         vqmovun.s16     d2,  q1
1779         vst1.32         {d2[0]},[r4,:32], r1
1780         vst1.32         {d2[1]},[r4,:32], r1
1781         pop             {r4, pc}
1782 .endm
1783
1784 .macro  weight_func     w
1785 function ff_weight_h264_pixels_\w\()_neon, export=1
1786         push            {r4, lr}
1787         ldr             r12, [sp, #8]
1788         ldr             r4,  [sp, #12]
1789         cmp             r3,  #1
1790         lsl             r4,  r4,  r3
1791         vdup.16         q8,  r4
1792         mov             r4,  r0
1793         ble             20f
1794         rsb             lr,  r3,  #1
1795         vdup.16         q9,  lr
1796         cmp             r12, #0
1797         blt             10f
1798         weight_\w       vhadd.s16
1799 10:     rsb             r12, r12, #0
1800         weight_\w       vhsub.s16
1801 20:     rsb             lr,  r3,  #0
1802         vdup.16         q9,  lr
1803         cmp             r12, #0
1804         blt             10f
1805         weight_\w       vadd.s16
1806 10:     rsb             r12, r12, #0
1807         weight_\w       vsub.s16
1808 endfunc
1809 .endm
1810
1811         weight_func     16
1812         weight_func     8
1813         weight_func     4