arm: fix compile with disable-optimizations on android
[ffmpeg.git] / libavcodec / arm / h264dsp_neon.S
1 /*
2  * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #include "libavutil/arm/asm.S"
22 #include "neon.S"
23
24         /* H.264 loop filter */
25
26 .macro  h264_loop_filter_start
27         ldr             r12, [sp]
28         tst             r2,  r2
29         ldr             r12, [r12]
30         it              ne
31         tstne           r3,  r3
32         vmov.32         d24[0], r12
33         and             r12, r12, r12, lsl #16
34         it              eq
35         bxeq            lr
36         ands            r12, r12, r12, lsl #8
37         it              lt
38         bxlt            lr
39 .endm
40
41 .macro  h264_loop_filter_luma
42         vdup.8          q11, r2         @ alpha
43         vmovl.u8        q12, d24
44         vabd.u8         q6,  q8,  q0    @ abs(p0 - q0)
45         vmovl.u16       q12, d24
46         vabd.u8         q14, q9,  q8    @ abs(p1 - p0)
47         vsli.16         q12, q12, #8
48         vabd.u8         q15, q1,  q0    @ abs(q1 - q0)
49         vsli.32         q12, q12, #16
50         vclt.u8         q6,  q6,  q11   @ < alpha
51         vdup.8          q11, r3         @ beta
52         vclt.s8         q7,  q12, #0
53         vclt.u8         q14, q14, q11   @ < beta
54         vclt.u8         q15, q15, q11   @ < beta
55         vbic            q6,  q6,  q7
56         vabd.u8         q4,  q10, q8    @ abs(p2 - p0)
57         vand            q6,  q6,  q14
58         vabd.u8         q5,  q2,  q0    @ abs(q2 - q0)
59         vclt.u8         q4,  q4,  q11   @ < beta
60         vand            q6,  q6,  q15
61         vclt.u8         q5,  q5,  q11   @ < beta
62         vand            q4,  q4,  q6
63         vand            q5,  q5,  q6
64         vand            q12, q12, q6
65         vrhadd.u8       q14, q8,  q0
66         vsub.i8         q6,  q12, q4
67         vqadd.u8        q7,  q9,  q12
68         vhadd.u8        q10, q10, q14
69         vsub.i8         q6,  q6,  q5
70         vhadd.u8        q14, q2,  q14
71         vmin.u8         q7,  q7,  q10
72         vqsub.u8        q11, q9,  q12
73         vqadd.u8        q2,  q1,  q12
74         vmax.u8         q7,  q7,  q11
75         vqsub.u8        q11, q1,  q12
76         vmin.u8         q14, q2,  q14
77         vmovl.u8        q2,  d0
78         vmax.u8         q14, q14, q11
79         vmovl.u8        q10, d1
80         vsubw.u8        q2,  q2,  d16
81         vsubw.u8        q10, q10, d17
82         vshl.i16        q2,  q2,  #2
83         vshl.i16        q10, q10, #2
84         vaddw.u8        q2,  q2,  d18
85         vaddw.u8        q10, q10, d19
86         vsubw.u8        q2,  q2,  d2
87         vsubw.u8        q10, q10, d3
88         vrshrn.i16      d4,  q2,  #3
89         vrshrn.i16      d5,  q10, #3
90         vbsl            q4,  q7,  q9
91         vbsl            q5,  q14, q1
92         vneg.s8         q7,  q6
93         vmovl.u8        q14, d16
94         vmin.s8         q2,  q2,  q6
95         vmovl.u8        q6,  d17
96         vmax.s8         q2,  q2,  q7
97         vmovl.u8        q11, d0
98         vmovl.u8        q12, d1
99         vaddw.s8        q14, q14, d4
100         vaddw.s8        q6,  q6,  d5
101         vsubw.s8        q11, q11, d4
102         vsubw.s8        q12, q12, d5
103         vqmovun.s16     d16, q14
104         vqmovun.s16     d17, q6
105         vqmovun.s16     d0,  q11
106         vqmovun.s16     d1,  q12
107 .endm
108
109 function ff_h264_v_loop_filter_luma_neon, export=1
110         h264_loop_filter_start
111
112         vld1.8          {d0, d1},  [r0,:128], r1
113         vld1.8          {d2, d3},  [r0,:128], r1
114         vld1.8          {d4, d5},  [r0,:128], r1
115         sub             r0,  r0,  r1, lsl #2
116         sub             r0,  r0,  r1, lsl #1
117         vld1.8          {d20,d21}, [r0,:128], r1
118         vld1.8          {d18,d19}, [r0,:128], r1
119         vld1.8          {d16,d17}, [r0,:128], r1
120
121         vpush           {d8-d15}
122
123         h264_loop_filter_luma
124
125         sub             r0,  r0,  r1, lsl #1
126         vst1.8          {d8, d9},  [r0,:128], r1
127         vst1.8          {d16,d17}, [r0,:128], r1
128         vst1.8          {d0, d1},  [r0,:128], r1
129         vst1.8          {d10,d11}, [r0,:128]
130
131         vpop            {d8-d15}
132         bx              lr
133 endfunc
134
135 function ff_h264_h_loop_filter_luma_neon, export=1
136         h264_loop_filter_start
137
138         sub             r0,  r0,  #4
139         vld1.8          {d6},  [r0], r1
140         vld1.8          {d20}, [r0], r1
141         vld1.8          {d18}, [r0], r1
142         vld1.8          {d16}, [r0], r1
143         vld1.8          {d0},  [r0], r1
144         vld1.8          {d2},  [r0], r1
145         vld1.8          {d4},  [r0], r1
146         vld1.8          {d26}, [r0], r1
147         vld1.8          {d7},  [r0], r1
148         vld1.8          {d21}, [r0], r1
149         vld1.8          {d19}, [r0], r1
150         vld1.8          {d17}, [r0], r1
151         vld1.8          {d1},  [r0], r1
152         vld1.8          {d3},  [r0], r1
153         vld1.8          {d5},  [r0], r1
154         vld1.8          {d27}, [r0], r1
155
156         transpose_8x8   q3, q10, q9, q8, q0, q1, q2, q13
157
158         vpush           {d8-d15}
159
160         h264_loop_filter_luma
161
162         transpose_4x4   q4, q8, q0, q5
163
164         sub             r0,  r0,  r1, lsl #4
165         add             r0,  r0,  #2
166         vst1.32         {d8[0]},  [r0], r1
167         vst1.32         {d16[0]}, [r0], r1
168         vst1.32         {d0[0]},  [r0], r1
169         vst1.32         {d10[0]}, [r0], r1
170         vst1.32         {d8[1]},  [r0], r1
171         vst1.32         {d16[1]}, [r0], r1
172         vst1.32         {d0[1]},  [r0], r1
173         vst1.32         {d10[1]}, [r0], r1
174         vst1.32         {d9[0]},  [r0], r1
175         vst1.32         {d17[0]}, [r0], r1
176         vst1.32         {d1[0]},  [r0], r1
177         vst1.32         {d11[0]}, [r0], r1
178         vst1.32         {d9[1]},  [r0], r1
179         vst1.32         {d17[1]}, [r0], r1
180         vst1.32         {d1[1]},  [r0], r1
181         vst1.32         {d11[1]}, [r0], r1
182
183         vpop            {d8-d15}
184         bx              lr
185 endfunc
186
187 .macro  h264_loop_filter_chroma
188         vdup.8          d22, r2         @ alpha
189         vmovl.u8        q12, d24
190         vabd.u8         d26, d16, d0    @ abs(p0 - q0)
191         vmovl.u8        q2,  d0
192         vabd.u8         d28, d18, d16   @ abs(p1 - p0)
193         vsubw.u8        q2,  q2,  d16
194         vsli.16         d24, d24, #8
195         vshl.i16        q2,  q2,  #2
196         vabd.u8         d30, d2,  d0    @ abs(q1 - q0)
197         vaddw.u8        q2,  q2,  d18
198         vclt.u8         d26, d26, d22   @ < alpha
199         vsubw.u8        q2,  q2,  d2
200         vdup.8          d22, r3         @ beta
201         vrshrn.i16      d4,  q2,  #3
202         vclt.u8         d28, d28, d22   @ < beta
203         vclt.u8         d30, d30, d22   @ < beta
204         vmin.s8         d4,  d4,  d24
205         vneg.s8         d25, d24
206         vand            d26, d26, d28
207         vmax.s8         d4,  d4,  d25
208         vand            d26, d26, d30
209         vmovl.u8        q11, d0
210         vand            d4,  d4,  d26
211         vmovl.u8        q14, d16
212         vaddw.s8        q14, q14, d4
213         vsubw.s8        q11, q11, d4
214         vqmovun.s16     d16, q14
215         vqmovun.s16     d0,  q11
216 .endm
217
218 function ff_h264_v_loop_filter_chroma_neon, export=1
219         h264_loop_filter_start
220
221         sub             r0,  r0,  r1, lsl #1
222         vld1.8          {d18}, [r0,:64], r1
223         vld1.8          {d16}, [r0,:64], r1
224         vld1.8          {d0},  [r0,:64], r1
225         vld1.8          {d2},  [r0,:64]
226
227         h264_loop_filter_chroma
228
229         sub             r0,  r0,  r1, lsl #1
230         vst1.8          {d16}, [r0,:64], r1
231         vst1.8          {d0},  [r0,:64], r1
232
233         bx              lr
234 endfunc
235
236 function ff_h264_h_loop_filter_chroma_neon, export=1
237         h264_loop_filter_start
238
239         sub             r0,  r0,  #2
240         vld1.32         {d18[0]}, [r0], r1
241         vld1.32         {d16[0]}, [r0], r1
242         vld1.32         {d0[0]},  [r0], r1
243         vld1.32         {d2[0]},  [r0], r1
244         vld1.32         {d18[1]}, [r0], r1
245         vld1.32         {d16[1]}, [r0], r1
246         vld1.32         {d0[1]},  [r0], r1
247         vld1.32         {d2[1]},  [r0], r1
248
249         vtrn.16         d18, d0
250         vtrn.16         d16, d2
251         vtrn.8          d18, d16
252         vtrn.8          d0,  d2
253
254         h264_loop_filter_chroma
255
256         vtrn.16         d18, d0
257         vtrn.16         d16, d2
258         vtrn.8          d18, d16
259         vtrn.8          d0,  d2
260
261         sub             r0,  r0,  r1, lsl #3
262         vst1.32         {d18[0]}, [r0], r1
263         vst1.32         {d16[0]}, [r0], r1
264         vst1.32         {d0[0]},  [r0], r1
265         vst1.32         {d2[0]},  [r0], r1
266         vst1.32         {d18[1]}, [r0], r1
267         vst1.32         {d16[1]}, [r0], r1
268         vst1.32         {d0[1]},  [r0], r1
269         vst1.32         {d2[1]},  [r0], r1
270
271         bx              lr
272 endfunc
273
274         /* H.264 qpel MC */
275
276 .macro  lowpass_const   r
277         movw            \r,  #5
278         movt            \r,  #20
279         vmov.32         d6[0], \r
280 .endm
281
282 .macro  lowpass_8       r0,  r1,  r2,  r3,  d0,  d1,  narrow=1
283   .if \narrow
284         t0 .req q0
285         t1 .req q8
286   .else
287         t0 .req \d0
288         t1 .req \d1
289   .endif
290         vext.8          d2,  \r0, \r1, #2
291         vext.8          d3,  \r0, \r1, #3
292         vaddl.u8        q1,  d2,  d3
293         vext.8          d4,  \r0, \r1, #1
294         vext.8          d5,  \r0, \r1, #4
295         vaddl.u8        q2,  d4,  d5
296         vext.8          d30, \r0, \r1, #5
297         vaddl.u8        t0,  \r0, d30
298         vext.8          d18, \r2, \r3, #2
299         vmla.i16        t0,  q1,  d6[1]
300         vext.8          d19, \r2, \r3, #3
301         vaddl.u8        q9,  d18, d19
302         vext.8          d20, \r2, \r3, #1
303         vmls.i16        t0,  q2,  d6[0]
304         vext.8          d21, \r2, \r3, #4
305         vaddl.u8        q10, d20, d21
306         vext.8          d31, \r2, \r3, #5
307         vaddl.u8        t1,  \r2, d31
308         vmla.i16        t1,  q9,  d6[1]
309         vmls.i16        t1,  q10, d6[0]
310   .if \narrow
311         vqrshrun.s16    \d0, t0,  #5
312         vqrshrun.s16    \d1, t1,  #5
313   .endif
314         .unreq  t0
315         .unreq  t1
316 .endm
317
318 .macro  lowpass_8_1     r0,  r1,  d0,  narrow=1
319   .if \narrow
320         t0 .req q0
321   .else
322         t0 .req \d0
323   .endif
324         vext.8          d2,  \r0, \r1, #2
325         vext.8          d3,  \r0, \r1, #3
326         vaddl.u8        q1,  d2,  d3
327         vext.8          d4,  \r0, \r1, #1
328         vext.8          d5,  \r0, \r1, #4
329         vaddl.u8        q2,  d4,  d5
330         vext.8          d30, \r0, \r1, #5
331         vaddl.u8        t0,  \r0, d30
332         vmla.i16        t0,  q1,  d6[1]
333         vmls.i16        t0,  q2,  d6[0]
334   .if \narrow
335         vqrshrun.s16    \d0, t0,  #5
336   .endif
337         .unreq  t0
338 .endm
339
340 .macro  lowpass_8.16    r0,  r1,  l0,  h0,  l1,  h1,  d
341         vext.16         q1,  \r0, \r1, #2
342         vext.16         q0,  \r0, \r1, #3
343         vaddl.s16       q9,  d2,  d0
344         vext.16         q2,  \r0, \r1, #1
345         vaddl.s16       q1,  d3,  d1
346         vext.16         q3,  \r0, \r1, #4
347         vaddl.s16       q10, d4,  d6
348         vext.16         \r1, \r0, \r1, #5
349         vaddl.s16       q2,  d5,  d7
350         vaddl.s16       q0,  \h0, \h1
351         vaddl.s16       q8,  \l0, \l1
352
353         vshl.i32        q3,  q9,  #4
354         vshl.i32        q9,  q9,  #2
355         vshl.i32        q15, q10, #2
356         vadd.i32        q9,  q9,  q3
357         vadd.i32        q10, q10, q15
358
359         vshl.i32        q3,  q1,  #4
360         vshl.i32        q1,  q1,  #2
361         vshl.i32        q15, q2,  #2
362         vadd.i32        q1,  q1,  q3
363         vadd.i32        q2,  q2,  q15
364
365         vadd.i32        q9,  q9,  q8
366         vsub.i32        q9,  q9,  q10
367
368         vadd.i32        q1,  q1,  q0
369         vsub.i32        q1,  q1,  q2
370
371         vrshrn.s32      d18, q9,  #10
372         vrshrn.s32      d19, q1,  #10
373
374         vqmovun.s16     \d,  q9
375 .endm
376
377 function put_h264_qpel16_h_lowpass_neon_packed
378         mov             r4,  lr
379         mov             r12, #16
380         mov             r3,  #8
381         bl              put_h264_qpel8_h_lowpass_neon
382         sub             r1,  r1,  r2, lsl #4
383         add             r1,  r1,  #8
384         mov             r12, #16
385         mov             lr,  r4
386         b               put_h264_qpel8_h_lowpass_neon
387 endfunc
388
389 .macro  h264_qpel_h_lowpass type
390 function \type\()_h264_qpel16_h_lowpass_neon
391         push            {lr}
392         mov             r12, #16
393         bl              \type\()_h264_qpel8_h_lowpass_neon
394         sub             r0,  r0,  r3, lsl #4
395         sub             r1,  r1,  r2, lsl #4
396         add             r0,  r0,  #8
397         add             r1,  r1,  #8
398         mov             r12, #16
399         pop             {lr}
400 endfunc
401
402 function \type\()_h264_qpel8_h_lowpass_neon
403 1:      vld1.8          {d0, d1},  [r1], r2
404         vld1.8          {d16,d17}, [r1], r2
405         subs            r12, r12, #2
406         lowpass_8       d0,  d1,  d16, d17, d0,  d16
407   .ifc \type,avg
408         vld1.8          {d2},     [r0,:64], r3
409         vrhadd.u8       d0,  d0,  d2
410         vld1.8          {d3},     [r0,:64]
411         vrhadd.u8       d16, d16, d3
412         sub             r0,  r0,  r3
413   .endif
414         vst1.8          {d0},     [r0,:64], r3
415         vst1.8          {d16},    [r0,:64], r3
416         bne             1b
417         bx              lr
418 endfunc
419 .endm
420
421         h264_qpel_h_lowpass put
422         h264_qpel_h_lowpass avg
423
424 .macro  h264_qpel_h_lowpass_l2 type
425 function \type\()_h264_qpel16_h_lowpass_l2_neon
426         push            {lr}
427         mov             r12, #16
428         bl              \type\()_h264_qpel8_h_lowpass_l2_neon
429         sub             r0,  r0,  r2, lsl #4
430         sub             r1,  r1,  r2, lsl #4
431         sub             r3,  r3,  r2, lsl #4
432         add             r0,  r0,  #8
433         add             r1,  r1,  #8
434         add             r3,  r3,  #8
435         mov             r12, #16
436         pop             {lr}
437 endfunc
438
439 function \type\()_h264_qpel8_h_lowpass_l2_neon
440 1:      vld1.8          {d0, d1},  [r1], r2
441         vld1.8          {d16,d17}, [r1], r2
442         vld1.8          {d28},     [r3], r2
443         vld1.8          {d29},     [r3], r2
444         subs            r12, r12, #2
445         lowpass_8       d0,  d1,  d16, d17, d0,  d1
446         vrhadd.u8       q0,  q0,  q14
447   .ifc \type,avg
448         vld1.8          {d2},      [r0,:64], r2
449         vrhadd.u8       d0,  d0,  d2
450         vld1.8          {d3},      [r0,:64]
451         vrhadd.u8       d1,  d1,  d3
452         sub             r0,  r0,  r2
453   .endif
454         vst1.8          {d0},      [r0,:64], r2
455         vst1.8          {d1},      [r0,:64], r2
456         bne             1b
457         bx              lr
458 endfunc
459 .endm
460
461         h264_qpel_h_lowpass_l2 put
462         h264_qpel_h_lowpass_l2 avg
463
464 function put_h264_qpel16_v_lowpass_neon_packed
465         mov             r4,  lr
466         mov             r2,  #8
467         bl              put_h264_qpel8_v_lowpass_neon
468         sub             r1,  r1,  r3, lsl #2
469         bl              put_h264_qpel8_v_lowpass_neon
470         sub             r1,  r1,  r3, lsl #4
471         sub             r1,  r1,  r3, lsl #2
472         add             r1,  r1,  #8
473         bl              put_h264_qpel8_v_lowpass_neon
474         sub             r1,  r1,  r3, lsl #2
475         mov             lr,  r4
476         b               put_h264_qpel8_v_lowpass_neon
477 endfunc
478
479 .macro  h264_qpel_v_lowpass type
480 function \type\()_h264_qpel16_v_lowpass_neon
481         mov             r4,  lr
482         bl              \type\()_h264_qpel8_v_lowpass_neon
483         sub             r1,  r1,  r3, lsl #2
484         bl              \type\()_h264_qpel8_v_lowpass_neon
485         sub             r0,  r0,  r2, lsl #4
486         add             r0,  r0,  #8
487         sub             r1,  r1,  r3, lsl #4
488         sub             r1,  r1,  r3, lsl #2
489         add             r1,  r1,  #8
490         bl              \type\()_h264_qpel8_v_lowpass_neon
491         sub             r1,  r1,  r3, lsl #2
492         mov             lr,  r4
493 endfunc
494
495 function \type\()_h264_qpel8_v_lowpass_neon
496         vld1.8          {d8},  [r1], r3
497         vld1.8          {d10}, [r1], r3
498         vld1.8          {d12}, [r1], r3
499         vld1.8          {d14}, [r1], r3
500         vld1.8          {d22}, [r1], r3
501         vld1.8          {d24}, [r1], r3
502         vld1.8          {d26}, [r1], r3
503         vld1.8          {d28}, [r1], r3
504         vld1.8          {d9},  [r1], r3
505         vld1.8          {d11}, [r1], r3
506         vld1.8          {d13}, [r1], r3
507         vld1.8          {d15}, [r1], r3
508         vld1.8          {d23}, [r1]
509
510         transpose_8x8   q4,  q5,  q6,  q7,  q11, q12, q13, q14
511         lowpass_8       d8,  d9,  d10, d11, d8,  d10
512         lowpass_8       d12, d13, d14, d15, d12, d14
513         lowpass_8       d22, d23, d24, d25, d22, d24
514         lowpass_8       d26, d27, d28, d29, d26, d28
515         transpose_8x8   d8,  d10, d12, d14, d22, d24, d26, d28
516
517   .ifc \type,avg
518         vld1.8          {d9},  [r0,:64], r2
519         vrhadd.u8       d8,  d8,  d9
520         vld1.8          {d11}, [r0,:64], r2
521         vrhadd.u8       d10, d10, d11
522         vld1.8          {d13}, [r0,:64], r2
523         vrhadd.u8       d12, d12, d13
524         vld1.8          {d15}, [r0,:64], r2
525         vrhadd.u8       d14, d14, d15
526         vld1.8          {d23}, [r0,:64], r2
527         vrhadd.u8       d22, d22, d23
528         vld1.8          {d25}, [r0,:64], r2
529         vrhadd.u8       d24, d24, d25
530         vld1.8          {d27}, [r0,:64], r2
531         vrhadd.u8       d26, d26, d27
532         vld1.8          {d29}, [r0,:64], r2
533         vrhadd.u8       d28, d28, d29
534         sub             r0,  r0,  r2,  lsl #3
535   .endif
536
537         vst1.8          {d8},  [r0,:64], r2
538         vst1.8          {d10}, [r0,:64], r2
539         vst1.8          {d12}, [r0,:64], r2
540         vst1.8          {d14}, [r0,:64], r2
541         vst1.8          {d22}, [r0,:64], r2
542         vst1.8          {d24}, [r0,:64], r2
543         vst1.8          {d26}, [r0,:64], r2
544         vst1.8          {d28}, [r0,:64], r2
545
546         bx              lr
547 endfunc
548 .endm
549
550         h264_qpel_v_lowpass put
551         h264_qpel_v_lowpass avg
552
553 .macro  h264_qpel_v_lowpass_l2 type
554 function \type\()_h264_qpel16_v_lowpass_l2_neon
555         mov             r4,  lr
556         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
557         sub             r1,  r1,  r3, lsl #2
558         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
559         sub             r0,  r0,  r3, lsl #4
560         sub             r12, r12, r2, lsl #4
561         add             r0,  r0,  #8
562         add             r12, r12, #8
563         sub             r1,  r1,  r3, lsl #4
564         sub             r1,  r1,  r3, lsl #2
565         add             r1,  r1,  #8
566         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
567         sub             r1,  r1,  r3, lsl #2
568         mov             lr,  r4
569 endfunc
570
571 function \type\()_h264_qpel8_v_lowpass_l2_neon
572         vld1.8          {d8},  [r1], r3
573         vld1.8          {d10}, [r1], r3
574         vld1.8          {d12}, [r1], r3
575         vld1.8          {d14}, [r1], r3
576         vld1.8          {d22}, [r1], r3
577         vld1.8          {d24}, [r1], r3
578         vld1.8          {d26}, [r1], r3
579         vld1.8          {d28}, [r1], r3
580         vld1.8          {d9},  [r1], r3
581         vld1.8          {d11}, [r1], r3
582         vld1.8          {d13}, [r1], r3
583         vld1.8          {d15}, [r1], r3
584         vld1.8          {d23}, [r1]
585
586         transpose_8x8   q4,  q5,  q6,  q7,  q11, q12, q13, q14
587         lowpass_8       d8,  d9,  d10, d11, d8,  d9
588         lowpass_8       d12, d13, d14, d15, d12, d13
589         lowpass_8       d22, d23, d24, d25, d22, d23
590         lowpass_8       d26, d27, d28, d29, d26, d27
591         transpose_8x8   d8,  d9,  d12, d13, d22, d23, d26, d27
592
593         vld1.8          {d0},  [r12], r2
594         vld1.8          {d1},  [r12], r2
595         vld1.8          {d2},  [r12], r2
596         vld1.8          {d3},  [r12], r2
597         vld1.8          {d4},  [r12], r2
598         vrhadd.u8       q0,  q0,  q4
599         vld1.8          {d5},  [r12], r2
600         vrhadd.u8       q1,  q1,  q6
601         vld1.8          {d10}, [r12], r2
602         vrhadd.u8       q2,  q2,  q11
603         vld1.8          {d11}, [r12], r2
604         vrhadd.u8       q5,  q5,  q13
605
606   .ifc \type,avg
607         vld1.8          {d16}, [r0,:64], r3
608         vrhadd.u8       d0,  d0,  d16
609         vld1.8          {d17}, [r0,:64], r3
610         vrhadd.u8       d1,  d1,  d17
611         vld1.8          {d16}, [r0,:64], r3
612         vrhadd.u8       d2,  d2,  d16
613         vld1.8          {d17}, [r0,:64], r3
614         vrhadd.u8       d3,  d3,  d17
615         vld1.8          {d16}, [r0,:64], r3
616         vrhadd.u8       d4,  d4,  d16
617         vld1.8          {d17}, [r0,:64], r3
618         vrhadd.u8       d5,  d5,  d17
619         vld1.8          {d16}, [r0,:64], r3
620         vrhadd.u8       d10, d10, d16
621         vld1.8          {d17}, [r0,:64], r3
622         vrhadd.u8       d11, d11, d17
623         sub             r0,  r0,  r3,  lsl #3
624   .endif
625
626         vst1.8          {d0},  [r0,:64], r3
627         vst1.8          {d1},  [r0,:64], r3
628         vst1.8          {d2},  [r0,:64], r3
629         vst1.8          {d3},  [r0,:64], r3
630         vst1.8          {d4},  [r0,:64], r3
631         vst1.8          {d5},  [r0,:64], r3
632         vst1.8          {d10}, [r0,:64], r3
633         vst1.8          {d11}, [r0,:64], r3
634
635         bx              lr
636 endfunc
637 .endm
638
639         h264_qpel_v_lowpass_l2 put
640         h264_qpel_v_lowpass_l2 avg
641
642 function put_h264_qpel8_hv_lowpass_neon_top
643         lowpass_const   r12
644         mov             r12, #12
645 1:      vld1.8          {d0, d1},  [r1], r3
646         vld1.8          {d16,d17}, [r1], r3
647         subs            r12, r12, #2
648         lowpass_8       d0,  d1,  d16, d17, q11, q12, narrow=0
649         vst1.8          {d22-d25}, [r4,:128]!
650         bne             1b
651
652         vld1.8          {d0, d1},  [r1]
653         lowpass_8_1     d0,  d1,  q12, narrow=0
654
655         mov             r12, #-16
656         add             r4,  r4,  r12
657         vld1.8          {d30,d31}, [r4,:128], r12
658         vld1.8          {d20,d21}, [r4,:128], r12
659         vld1.8          {d18,d19}, [r4,:128], r12
660         vld1.8          {d16,d17}, [r4,:128], r12
661         vld1.8          {d14,d15}, [r4,:128], r12
662         vld1.8          {d12,d13}, [r4,:128], r12
663         vld1.8          {d10,d11}, [r4,:128], r12
664         vld1.8          {d8, d9},  [r4,:128], r12
665         vld1.8          {d6, d7},  [r4,:128], r12
666         vld1.8          {d4, d5},  [r4,:128], r12
667         vld1.8          {d2, d3},  [r4,:128], r12
668         vld1.8          {d0, d1},  [r4,:128]
669
670         swap4           d1,  d3,  d5,  d7,  d8,  d10, d12, d14
671         transpose16_4x4 q0,  q1,  q2,  q3,  q4,  q5,  q6,  q7
672
673         swap4           d17, d19, d21, d31, d24, d26, d28, d22
674         transpose16_4x4 q8,  q9,  q10, q15, q12, q13, q14, q11
675
676         vst1.8          {d30,d31}, [r4,:128]!
677         vst1.8          {d6, d7},  [r4,:128]!
678         vst1.8          {d20,d21}, [r4,:128]!
679         vst1.8          {d4, d5},  [r4,:128]!
680         vst1.8          {d18,d19}, [r4,:128]!
681         vst1.8          {d2, d3},  [r4,:128]!
682         vst1.8          {d16,d17}, [r4,:128]!
683         vst1.8          {d0, d1},  [r4,:128]
684
685         lowpass_8.16    q4,  q12, d8,  d9,  d24, d25, d8
686         lowpass_8.16    q5,  q13, d10, d11, d26, d27, d9
687         lowpass_8.16    q6,  q14, d12, d13, d28, d29, d10
688         lowpass_8.16    q7,  q11, d14, d15, d22, d23, d11
689
690         vld1.8          {d16,d17}, [r4,:128], r12
691         vld1.8          {d30,d31}, [r4,:128], r12
692         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d12
693         vld1.8          {d16,d17}, [r4,:128], r12
694         vld1.8          {d30,d31}, [r4,:128], r12
695         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d13
696         vld1.8          {d16,d17}, [r4,:128], r12
697         vld1.8          {d30,d31}, [r4,:128], r12
698         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d14
699         vld1.8          {d16,d17}, [r4,:128], r12
700         vld1.8          {d30,d31}, [r4,:128]
701         lowpass_8.16    q8,  q15, d16, d17, d30, d31, d15
702
703         transpose_8x8   d12, d13, d14, d15, d8,  d9,  d10, d11
704
705         bx              lr
706 endfunc
707
708 .macro  h264_qpel8_hv_lowpass type
709 function \type\()_h264_qpel8_hv_lowpass_neon
710         mov             r10, lr
711         bl              put_h264_qpel8_hv_lowpass_neon_top
712   .ifc \type,avg
713         vld1.8          {d0},      [r0,:64], r2
714         vrhadd.u8       d12, d12, d0
715         vld1.8          {d1},      [r0,:64], r2
716         vrhadd.u8       d13, d13, d1
717         vld1.8          {d2},      [r0,:64], r2
718         vrhadd.u8       d14, d14, d2
719         vld1.8          {d3},      [r0,:64], r2
720         vrhadd.u8       d15, d15, d3
721         vld1.8          {d4},      [r0,:64], r2
722         vrhadd.u8       d8,  d8,  d4
723         vld1.8          {d5},      [r0,:64], r2
724         vrhadd.u8       d9,  d9,  d5
725         vld1.8          {d6},      [r0,:64], r2
726         vrhadd.u8       d10, d10, d6
727         vld1.8          {d7},      [r0,:64], r2
728         vrhadd.u8       d11, d11, d7
729         sub             r0,  r0,  r2,  lsl #3
730   .endif
731
732         vst1.8          {d12},     [r0,:64], r2
733         vst1.8          {d13},     [r0,:64], r2
734         vst1.8          {d14},     [r0,:64], r2
735         vst1.8          {d15},     [r0,:64], r2
736         vst1.8          {d8},      [r0,:64], r2
737         vst1.8          {d9},      [r0,:64], r2
738         vst1.8          {d10},     [r0,:64], r2
739         vst1.8          {d11},     [r0,:64], r2
740
741         mov             lr,  r10
742         bx              lr
743 endfunc
744 .endm
745
746         h264_qpel8_hv_lowpass put
747         h264_qpel8_hv_lowpass avg
748
749 .macro  h264_qpel8_hv_lowpass_l2 type
750 function \type\()_h264_qpel8_hv_lowpass_l2_neon
751         mov             r10, lr
752         bl              put_h264_qpel8_hv_lowpass_neon_top
753
754         vld1.8          {d0, d1},  [r2,:128]!
755         vld1.8          {d2, d3},  [r2,:128]!
756         vrhadd.u8       q0,  q0,  q6
757         vld1.8          {d4, d5},  [r2,:128]!
758         vrhadd.u8       q1,  q1,  q7
759         vld1.8          {d6, d7},  [r2,:128]!
760         vrhadd.u8       q2,  q2,  q4
761         vrhadd.u8       q3,  q3,  q5
762   .ifc \type,avg
763         vld1.8          {d16},     [r0,:64], r3
764         vrhadd.u8       d0,  d0,  d16
765         vld1.8          {d17},     [r0,:64], r3
766         vrhadd.u8       d1,  d1,  d17
767         vld1.8          {d18},     [r0,:64], r3
768         vrhadd.u8       d2,  d2,  d18
769         vld1.8          {d19},     [r0,:64], r3
770         vrhadd.u8       d3,  d3,  d19
771         vld1.8          {d20},     [r0,:64], r3
772         vrhadd.u8       d4,  d4,  d20
773         vld1.8          {d21},     [r0,:64], r3
774         vrhadd.u8       d5,  d5,  d21
775         vld1.8          {d22},     [r0,:64], r3
776         vrhadd.u8       d6,  d6,  d22
777         vld1.8          {d23},     [r0,:64], r3
778         vrhadd.u8       d7,  d7,  d23
779         sub             r0,  r0,  r3,  lsl #3
780   .endif
781         vst1.8          {d0},      [r0,:64], r3
782         vst1.8          {d1},      [r0,:64], r3
783         vst1.8          {d2},      [r0,:64], r3
784         vst1.8          {d3},      [r0,:64], r3
785         vst1.8          {d4},      [r0,:64], r3
786         vst1.8          {d5},      [r0,:64], r3
787         vst1.8          {d6},      [r0,:64], r3
788         vst1.8          {d7},      [r0,:64], r3
789
790         mov             lr,  r10
791         bx              lr
792 endfunc
793 .endm
794
795         h264_qpel8_hv_lowpass_l2 put
796         h264_qpel8_hv_lowpass_l2 avg
797
798 .macro  h264_qpel16_hv  type
799 function \type\()_h264_qpel16_hv_lowpass_neon
800         mov             r9,  lr
801         bl              \type\()_h264_qpel8_hv_lowpass_neon
802         sub             r1,  r1,  r3, lsl #2
803         bl              \type\()_h264_qpel8_hv_lowpass_neon
804         sub             r1,  r1,  r3, lsl #4
805         sub             r1,  r1,  r3, lsl #2
806         add             r1,  r1,  #8
807         sub             r0,  r0,  r2, lsl #4
808         add             r0,  r0,  #8
809         bl              \type\()_h264_qpel8_hv_lowpass_neon
810         sub             r1,  r1,  r3, lsl #2
811         mov             lr,  r9
812         b               \type\()_h264_qpel8_hv_lowpass_neon
813 endfunc
814
815 function \type\()_h264_qpel16_hv_lowpass_l2_neon
816         mov             r9,  lr
817         sub             r2,  r4,  #256
818         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
819         sub             r1,  r1,  r3, lsl #2
820         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
821         sub             r1,  r1,  r3, lsl #4
822         sub             r1,  r1,  r3, lsl #2
823         add             r1,  r1,  #8
824         sub             r0,  r0,  r3, lsl #4
825         add             r0,  r0,  #8
826         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
827         sub             r1,  r1,  r3, lsl #2
828         mov             lr,  r9
829         b               \type\()_h264_qpel8_hv_lowpass_l2_neon
830 endfunc
831 .endm
832
833         h264_qpel16_hv put
834         h264_qpel16_hv avg
835
836 .macro  h264_qpel8      type
837 function ff_\type\()_h264_qpel8_mc10_neon, export=1
838         lowpass_const   r3
839         mov             r3,  r1
840         sub             r1,  r1,  #2
841         mov             r12, #8
842         b               \type\()_h264_qpel8_h_lowpass_l2_neon
843 endfunc
844
845 function ff_\type\()_h264_qpel8_mc20_neon, export=1
846         lowpass_const   r3
847         sub             r1,  r1,  #2
848         mov             r3,  r2
849         mov             r12, #8
850         b               \type\()_h264_qpel8_h_lowpass_neon
851 endfunc
852
853 function ff_\type\()_h264_qpel8_mc30_neon, export=1
854         lowpass_const   r3
855         add             r3,  r1,  #1
856         sub             r1,  r1,  #2
857         mov             r12, #8
858         b               \type\()_h264_qpel8_h_lowpass_l2_neon
859 endfunc
860
861 function ff_\type\()_h264_qpel8_mc01_neon, export=1
862         push            {lr}
863         mov             r12, r1
864 \type\()_h264_qpel8_mc01:
865         lowpass_const   r3
866         mov             r3,  r2
867         sub             r1,  r1,  r2, lsl #1
868         vpush           {d8-d15}
869         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
870         vpop            {d8-d15}
871         pop             {pc}
872 endfunc
873
874 function ff_\type\()_h264_qpel8_mc11_neon, export=1
875         push            {r0, r1, r11, lr}
876 \type\()_h264_qpel8_mc11:
877         lowpass_const   r3
878         mov             r11, sp
879 A       bic             sp,  sp,  #15
880 T       bic             r0,  r11, #15
881 T       mov             sp,  r0
882         sub             sp,  sp,  #64
883         mov             r0,  sp
884         sub             r1,  r1,  #2
885         mov             r3,  #8
886         mov             r12, #8
887         vpush           {d8-d15}
888         bl              put_h264_qpel8_h_lowpass_neon
889         ldrd            r0,  r1,  [r11], #8
890         mov             r3,  r2
891         add             r12, sp,  #64
892         sub             r1,  r1,  r2, lsl #1
893         mov             r2,  #8
894         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
895         vpop            {d8-d15}
896         mov             sp,  r11
897         pop             {r11, pc}
898 endfunc
899
900 function ff_\type\()_h264_qpel8_mc21_neon, export=1
901         push            {r0, r1, r4, r10, r11, lr}
902 \type\()_h264_qpel8_mc21:
903         lowpass_const   r3
904         mov             r11, sp
905 A       bic             sp,  sp,  #15
906 T       bic             r0,  r11, #15
907 T       mov             sp,  r0
908         sub             sp,  sp,  #(8*8+16*12)
909         sub             r1,  r1,  #2
910         mov             r3,  #8
911         mov             r0,  sp
912         mov             r12, #8
913         vpush           {d8-d15}
914         bl              put_h264_qpel8_h_lowpass_neon
915         mov             r4,  r0
916         ldrd            r0,  r1,  [r11], #8
917         sub             r1,  r1,  r2, lsl #1
918         sub             r1,  r1,  #2
919         mov             r3,  r2
920         sub             r2,  r4,  #64
921         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
922         vpop            {d8-d15}
923         mov             sp,  r11
924         pop             {r4, r10, r11, pc}
925 endfunc
926
927 function ff_\type\()_h264_qpel8_mc31_neon, export=1
928         add             r1,  r1,  #1
929         push            {r0, r1, r11, lr}
930         sub             r1,  r1,  #1
931         b               \type\()_h264_qpel8_mc11
932 endfunc
933
934 function ff_\type\()_h264_qpel8_mc02_neon, export=1
935         push            {lr}
936         lowpass_const   r3
937         sub             r1,  r1,  r2, lsl #1
938         mov             r3,  r2
939         vpush           {d8-d15}
940         bl              \type\()_h264_qpel8_v_lowpass_neon
941         vpop            {d8-d15}
942         pop             {pc}
943 endfunc
944
945 function ff_\type\()_h264_qpel8_mc12_neon, export=1
946         push            {r0, r1, r4, r10, r11, lr}
947 \type\()_h264_qpel8_mc12:
948         lowpass_const   r3
949         mov             r11, sp
950 A       bic             sp,  sp,  #15
951 T       bic             r0,  r11, #15
952 T       mov             sp,  r0
953         sub             sp,  sp,  #(8*8+16*12)
954         sub             r1,  r1,  r2, lsl #1
955         mov             r3,  r2
956         mov             r2,  #8
957         mov             r0,  sp
958         vpush           {d8-d15}
959         bl              put_h264_qpel8_v_lowpass_neon
960         mov             r4,  r0
961         ldrd            r0,  r1,  [r11], #8
962         sub             r1,  r1,  r3, lsl #1
963         sub             r1,  r1,  #2
964         sub             r2,  r4,  #64
965         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
966         vpop            {d8-d15}
967         mov             sp,  r11
968         pop             {r4, r10, r11, pc}
969 endfunc
970
971 function ff_\type\()_h264_qpel8_mc22_neon, export=1
972         push            {r4, r10, r11, lr}
973         mov             r11, sp
974 A       bic             sp,  sp,  #15
975 T       bic             r4,  r11, #15
976 T       mov             sp,  r4
977         sub             r1,  r1,  r2, lsl #1
978         sub             r1,  r1,  #2
979         mov             r3,  r2
980         sub             sp,  sp,  #(16*12)
981         mov             r4,  sp
982         vpush           {d8-d15}
983         bl              \type\()_h264_qpel8_hv_lowpass_neon
984         vpop            {d8-d15}
985         mov             sp,  r11
986         pop             {r4, r10, r11, pc}
987 endfunc
988
989 function ff_\type\()_h264_qpel8_mc32_neon, export=1
990         push            {r0, r1, r4, r10, r11, lr}
991         add             r1,  r1,  #1
992         b               \type\()_h264_qpel8_mc12
993 endfunc
994
995 function ff_\type\()_h264_qpel8_mc03_neon, export=1
996         push            {lr}
997         add             r12, r1,  r2
998         b               \type\()_h264_qpel8_mc01
999 endfunc
1000
1001 function ff_\type\()_h264_qpel8_mc13_neon, export=1
1002         push            {r0, r1, r11, lr}
1003         add             r1,  r1,  r2
1004         b               \type\()_h264_qpel8_mc11
1005 endfunc
1006
1007 function ff_\type\()_h264_qpel8_mc23_neon, export=1
1008         push            {r0, r1, r4, r10, r11, lr}
1009         add             r1,  r1,  r2
1010         b               \type\()_h264_qpel8_mc21
1011 endfunc
1012
1013 function ff_\type\()_h264_qpel8_mc33_neon, export=1
1014         add             r1,  r1,  #1
1015         push            {r0, r1, r11, lr}
1016         add             r1,  r1,  r2
1017         sub             r1,  r1,  #1
1018         b               \type\()_h264_qpel8_mc11
1019 endfunc
1020 .endm
1021
1022         h264_qpel8 put
1023         h264_qpel8 avg
1024
1025 .macro  h264_qpel16     type
1026 function ff_\type\()_h264_qpel16_mc10_neon, export=1
1027         lowpass_const   r3
1028         mov             r3,  r1
1029         sub             r1,  r1,  #2
1030         b               \type\()_h264_qpel16_h_lowpass_l2_neon
1031 endfunc
1032
1033 function ff_\type\()_h264_qpel16_mc20_neon, export=1
1034         lowpass_const   r3
1035         sub             r1,  r1,  #2
1036         mov             r3,  r2
1037         b               \type\()_h264_qpel16_h_lowpass_neon
1038 endfunc
1039
1040 function ff_\type\()_h264_qpel16_mc30_neon, export=1
1041         lowpass_const   r3
1042         add             r3,  r1,  #1
1043         sub             r1,  r1,  #2
1044         b               \type\()_h264_qpel16_h_lowpass_l2_neon
1045 endfunc
1046
1047 function ff_\type\()_h264_qpel16_mc01_neon, export=1
1048         push            {r4, lr}
1049         mov             r12, r1
1050 \type\()_h264_qpel16_mc01:
1051         lowpass_const   r3
1052         mov             r3,  r2
1053         sub             r1,  r1,  r2, lsl #1
1054         vpush           {d8-d15}
1055         bl              \type\()_h264_qpel16_v_lowpass_l2_neon
1056         vpop            {d8-d15}
1057         pop             {r4, pc}
1058 endfunc
1059
1060 function ff_\type\()_h264_qpel16_mc11_neon, export=1
1061         push            {r0, r1, r4, r11, lr}
1062 \type\()_h264_qpel16_mc11:
1063         lowpass_const   r3
1064         mov             r11, sp
1065 A       bic             sp,  sp,  #15
1066 T       bic             r0,  r11, #15
1067 T       mov             sp,  r0
1068         sub             sp,  sp,  #256
1069         mov             r0,  sp
1070         sub             r1,  r1,  #2
1071         mov             r3,  #16
1072         vpush           {d8-d15}
1073         bl              put_h264_qpel16_h_lowpass_neon
1074         ldrd            r0,  r1,  [r11], #8
1075         mov             r3,  r2
1076         add             r12, sp,  #64
1077         sub             r1,  r1,  r2, lsl #1
1078         mov             r2,  #16
1079         bl              \type\()_h264_qpel16_v_lowpass_l2_neon
1080         vpop            {d8-d15}
1081         mov             sp,  r11
1082         pop             {r4, r11, pc}
1083 endfunc
1084
1085 function ff_\type\()_h264_qpel16_mc21_neon, export=1
1086         push            {r0, r1, r4-r5, r9-r11, lr}
1087 \type\()_h264_qpel16_mc21:
1088         lowpass_const   r3
1089         mov             r11, sp
1090 A       bic             sp,  sp,  #15
1091 T       bic             r0,  r11, #15
1092 T       mov             sp,  r0
1093         sub             sp,  sp,  #(16*16+16*12)
1094         sub             r1,  r1,  #2
1095         mov             r0,  sp
1096         vpush           {d8-d15}
1097         bl              put_h264_qpel16_h_lowpass_neon_packed
1098         mov             r4,  r0
1099         ldrd            r0,  r1,  [r11], #8
1100         sub             r1,  r1,  r2, lsl #1
1101         sub             r1,  r1,  #2
1102         mov             r3,  r2
1103         bl              \type\()_h264_qpel16_hv_lowpass_l2_neon
1104         vpop            {d8-d15}
1105         mov             sp,  r11
1106         pop             {r4-r5, r9-r11, pc}
1107 endfunc
1108
1109 function ff_\type\()_h264_qpel16_mc31_neon, export=1
1110         add             r1,  r1,  #1
1111         push            {r0, r1, r4, r11, lr}
1112         sub             r1,  r1,  #1
1113         b               \type\()_h264_qpel16_mc11
1114 endfunc
1115
1116 function ff_\type\()_h264_qpel16_mc02_neon, export=1
1117         push            {r4, lr}
1118         lowpass_const   r3
1119         sub             r1,  r1,  r2, lsl #1
1120         mov             r3,  r2
1121         vpush           {d8-d15}
1122         bl              \type\()_h264_qpel16_v_lowpass_neon
1123         vpop            {d8-d15}
1124         pop             {r4, pc}
1125 endfunc
1126
1127 function ff_\type\()_h264_qpel16_mc12_neon, export=1
1128         push            {r0, r1, r4-r5, r9-r11, lr}
1129 \type\()_h264_qpel16_mc12:
1130         lowpass_const   r3
1131         mov             r11, sp
1132 A       bic             sp,  sp,  #15
1133 T       bic             r0,  r11, #15
1134 T       mov             sp,  r0
1135         sub             sp,  sp,  #(16*16+16*12)
1136         sub             r1,  r1,  r2, lsl #1
1137         mov             r0,  sp
1138         mov             r3,  r2
1139         vpush           {d8-d15}
1140         bl              put_h264_qpel16_v_lowpass_neon_packed
1141         mov             r4,  r0
1142         ldrd            r0,  r1,  [r11], #8
1143         sub             r1,  r1,  r3, lsl #1
1144         sub             r1,  r1,  #2
1145         mov             r2,  r3
1146         bl              \type\()_h264_qpel16_hv_lowpass_l2_neon
1147         vpop            {d8-d15}
1148         mov             sp,  r11
1149         pop             {r4-r5, r9-r11, pc}
1150 endfunc
1151
1152 function ff_\type\()_h264_qpel16_mc22_neon, export=1
1153         push            {r4, r9-r11, lr}
1154         lowpass_const   r3
1155         mov             r11, sp
1156 A       bic             sp,  sp,  #15
1157 T       bic             r4,  r11, #15
1158 T       mov             sp,  r4
1159         sub             r1,  r1,  r2, lsl #1
1160         sub             r1,  r1,  #2
1161         mov             r3,  r2
1162         sub             sp,  sp,  #(16*12)
1163         mov             r4,  sp
1164         vpush           {d8-d15}
1165         bl              \type\()_h264_qpel16_hv_lowpass_neon
1166         vpop            {d8-d15}
1167         mov             sp,  r11
1168         pop             {r4, r9-r11, pc}
1169 endfunc
1170
1171 function ff_\type\()_h264_qpel16_mc32_neon, export=1
1172         push            {r0, r1, r4-r5, r9-r11, lr}
1173         add             r1,  r1,  #1
1174         b               \type\()_h264_qpel16_mc12
1175 endfunc
1176
1177 function ff_\type\()_h264_qpel16_mc03_neon, export=1
1178         push            {r4, lr}
1179         add             r12, r1,  r2
1180         b               \type\()_h264_qpel16_mc01
1181 endfunc
1182
1183 function ff_\type\()_h264_qpel16_mc13_neon, export=1
1184         push            {r0, r1, r4, r11, lr}
1185         add             r1,  r1,  r2
1186         b               \type\()_h264_qpel16_mc11
1187 endfunc
1188
1189 function ff_\type\()_h264_qpel16_mc23_neon, export=1
1190         push            {r0, r1, r4-r5, r9-r11, lr}
1191         add             r1,  r1,  r2
1192         b               \type\()_h264_qpel16_mc21
1193 endfunc
1194
1195 function ff_\type\()_h264_qpel16_mc33_neon, export=1
1196         add             r1,  r1,  #1
1197         push            {r0, r1, r4, r11, lr}
1198         add             r1,  r1,  r2
1199         sub             r1,  r1,  #1
1200         b               \type\()_h264_qpel16_mc11
1201 endfunc
1202 .endm
1203
1204         h264_qpel16 put
1205         h264_qpel16 avg
1206
1207 @ Biweighted prediction
1208
1209 .macro  biweight_16     macs, macd
1210         vdup.8          d0,  r4
1211         vdup.8          d1,  r5
1212         vmov            q2,  q8
1213         vmov            q3,  q8
1214 1:      subs            r3,  r3,  #2
1215         vld1.8          {d20-d21},[r0,:128], r2
1216         \macd           q2,  d0,  d20
1217         pld             [r0]
1218         \macd           q3,  d0,  d21
1219         vld1.8          {d22-d23},[r1,:128], r2
1220         \macs           q2,  d1,  d22
1221         pld             [r1]
1222         \macs           q3,  d1,  d23
1223         vmov            q12, q8
1224         vld1.8          {d28-d29},[r0,:128], r2
1225         vmov            q13, q8
1226         \macd           q12, d0,  d28
1227         pld             [r0]
1228         \macd           q13, d0,  d29
1229         vld1.8          {d30-d31},[r1,:128], r2
1230         \macs           q12, d1,  d30
1231         pld             [r1]
1232         \macs           q13, d1,  d31
1233         vshl.s16        q2,  q2,  q9
1234         vshl.s16        q3,  q3,  q9
1235         vqmovun.s16     d4,  q2
1236         vqmovun.s16     d5,  q3
1237         vshl.s16        q12, q12, q9
1238         vshl.s16        q13, q13, q9
1239         vqmovun.s16     d24, q12
1240         vqmovun.s16     d25, q13
1241         vmov            q3,  q8
1242         vst1.8          {d4- d5}, [r6,:128], r2
1243         vmov            q2,  q8
1244         vst1.8          {d24-d25},[r6,:128], r2
1245         bne             1b
1246         pop             {r4-r6, pc}
1247 .endm
1248
1249 .macro  biweight_8      macs, macd
1250         vdup.8          d0,  r4
1251         vdup.8          d1,  r5
1252         vmov            q1,  q8
1253         vmov            q10, q8
1254 1:      subs            r3,  r3,  #2
1255         vld1.8          {d4},[r0,:64], r2
1256         \macd           q1,  d0,  d4
1257         pld             [r0]
1258         vld1.8          {d5},[r1,:64], r2
1259         \macs           q1,  d1,  d5
1260         pld             [r1]
1261         vld1.8          {d6},[r0,:64], r2
1262         \macd           q10, d0,  d6
1263         pld             [r0]
1264         vld1.8          {d7},[r1,:64], r2
1265         \macs           q10, d1,  d7
1266         pld             [r1]
1267         vshl.s16        q1,  q1,  q9
1268         vqmovun.s16     d2,  q1
1269         vshl.s16        q10, q10, q9
1270         vqmovun.s16     d4,  q10
1271         vmov            q10, q8
1272         vst1.8          {d2},[r6,:64], r2
1273         vmov            q1,  q8
1274         vst1.8          {d4},[r6,:64], r2
1275         bne             1b
1276         pop             {r4-r6, pc}
1277 .endm
1278
1279 .macro  biweight_4      macs, macd
1280         vdup.8          d0,  r4
1281         vdup.8          d1,  r5
1282         vmov            q1,  q8
1283         vmov            q10, q8
1284 1:      subs            r3,  r3,  #4
1285         vld1.32         {d4[0]},[r0,:32], r2
1286         vld1.32         {d4[1]},[r0,:32], r2
1287         \macd           q1,  d0,  d4
1288         pld             [r0]
1289         vld1.32         {d5[0]},[r1,:32], r2
1290         vld1.32         {d5[1]},[r1,:32], r2
1291         \macs           q1,  d1,  d5
1292         pld             [r1]
1293         blt             2f
1294         vld1.32         {d6[0]},[r0,:32], r2
1295         vld1.32         {d6[1]},[r0,:32], r2
1296         \macd           q10, d0,  d6
1297         pld             [r0]
1298         vld1.32         {d7[0]},[r1,:32], r2
1299         vld1.32         {d7[1]},[r1,:32], r2
1300         \macs           q10, d1,  d7
1301         pld             [r1]
1302         vshl.s16        q1,  q1,  q9
1303         vqmovun.s16     d2,  q1
1304         vshl.s16        q10, q10, q9
1305         vqmovun.s16     d4,  q10
1306         vmov            q10, q8
1307         vst1.32         {d2[0]},[r6,:32], r2
1308         vst1.32         {d2[1]},[r6,:32], r2
1309         vmov            q1,  q8
1310         vst1.32         {d4[0]},[r6,:32], r2
1311         vst1.32         {d4[1]},[r6,:32], r2
1312         bne             1b
1313         pop             {r4-r6, pc}
1314 2:      vshl.s16        q1,  q1,  q9
1315         vqmovun.s16     d2,  q1
1316         vst1.32         {d2[0]},[r6,:32], r2
1317         vst1.32         {d2[1]},[r6,:32], r2
1318         pop             {r4-r6, pc}
1319 .endm
1320
1321 .macro  biweight_func   w
1322 function ff_biweight_h264_pixels_\w\()_neon, export=1
1323         push            {r4-r6, lr}
1324         ldr             r12, [sp, #16]
1325         add             r4,  sp,  #20
1326         ldm             r4,  {r4-r6}
1327         lsr             lr,  r4,  #31
1328         add             r6,  r6,  #1
1329         eors            lr,  lr,  r5,  lsr #30
1330         orr             r6,  r6,  #1
1331         vdup.16         q9,  r12
1332         lsl             r6,  r6,  r12
1333         vmvn            q9,  q9
1334         vdup.16         q8,  r6
1335         mov             r6,  r0
1336         beq             10f
1337         subs            lr,  lr,  #1
1338         beq             20f
1339         subs            lr,  lr,  #1
1340         beq             30f
1341         b               40f
1342 10:     biweight_\w     vmlal.u8, vmlal.u8
1343 20:     rsb             r4,  r4,  #0
1344         biweight_\w     vmlal.u8, vmlsl.u8
1345 30:     rsb             r4,  r4,  #0
1346         rsb             r5,  r5,  #0
1347         biweight_\w     vmlsl.u8, vmlsl.u8
1348 40:     rsb             r5,  r5,  #0
1349         biweight_\w     vmlsl.u8, vmlal.u8
1350 endfunc
1351 .endm
1352
1353         biweight_func   16
1354         biweight_func   8
1355         biweight_func   4
1356
1357 @ Weighted prediction
1358
1359 .macro  weight_16       add
1360         vdup.8          d0,  r12
1361 1:      subs            r2,  r2,  #2
1362         vld1.8          {d20-d21},[r0,:128], r1
1363         vmull.u8        q2,  d0,  d20
1364         pld             [r0]
1365         vmull.u8        q3,  d0,  d21
1366         vld1.8          {d28-d29},[r0,:128], r1
1367         vmull.u8        q12, d0,  d28
1368         pld             [r0]
1369         vmull.u8        q13, d0,  d29
1370         \add            q2,  q8,  q2
1371         vrshl.s16       q2,  q2,  q9
1372         \add            q3,  q8,  q3
1373         vrshl.s16       q3,  q3,  q9
1374         vqmovun.s16     d4,  q2
1375         vqmovun.s16     d5,  q3
1376         \add            q12, q8,  q12
1377         vrshl.s16       q12, q12, q9
1378         \add            q13, q8,  q13
1379         vrshl.s16       q13, q13, q9
1380         vqmovun.s16     d24, q12
1381         vqmovun.s16     d25, q13
1382         vst1.8          {d4- d5}, [r4,:128], r1
1383         vst1.8          {d24-d25},[r4,:128], r1
1384         bne             1b
1385         pop             {r4, pc}
1386 .endm
1387
1388 .macro  weight_8        add
1389         vdup.8          d0,  r12
1390 1:      subs            r2,  r2,  #2
1391         vld1.8          {d4},[r0,:64], r1
1392         vmull.u8        q1,  d0,  d4
1393         pld             [r0]
1394         vld1.8          {d6},[r0,:64], r1
1395         vmull.u8        q10, d0,  d6
1396         \add            q1,  q8,  q1
1397         pld             [r0]
1398         vrshl.s16       q1,  q1,  q9
1399         vqmovun.s16     d2,  q1
1400         \add            q10, q8,  q10
1401         vrshl.s16       q10, q10, q9
1402         vqmovun.s16     d4,  q10
1403         vst1.8          {d2},[r4,:64], r1
1404         vst1.8          {d4},[r4,:64], r1
1405         bne             1b
1406         pop             {r4, pc}
1407 .endm
1408
1409 .macro  weight_4        add
1410         vdup.8          d0,  r12
1411         vmov            q1,  q8
1412         vmov            q10, q8
1413 1:      subs            r2,  r2,  #4
1414         vld1.32         {d4[0]},[r0,:32], r1
1415         vld1.32         {d4[1]},[r0,:32], r1
1416         vmull.u8        q1,  d0,  d4
1417         pld             [r0]
1418         blt             2f
1419         vld1.32         {d6[0]},[r0,:32], r1
1420         vld1.32         {d6[1]},[r0,:32], r1
1421         vmull.u8        q10, d0,  d6
1422         pld             [r0]
1423         \add            q1,  q8,  q1
1424         vrshl.s16       q1,  q1,  q9
1425         vqmovun.s16     d2,  q1
1426         \add            q10, q8,  q10
1427         vrshl.s16       q10, q10, q9
1428         vqmovun.s16     d4,  q10
1429         vmov            q10, q8
1430         vst1.32         {d2[0]},[r4,:32], r1
1431         vst1.32         {d2[1]},[r4,:32], r1
1432         vmov            q1,  q8
1433         vst1.32         {d4[0]},[r4,:32], r1
1434         vst1.32         {d4[1]},[r4,:32], r1
1435         bne             1b
1436         pop             {r4, pc}
1437 2:      \add            q1,  q8,  q1
1438         vrshl.s16       q1,  q1,  q9
1439         vqmovun.s16     d2,  q1
1440         vst1.32         {d2[0]},[r4,:32], r1
1441         vst1.32         {d2[1]},[r4,:32], r1
1442         pop             {r4, pc}
1443 .endm
1444
1445 .macro  weight_func     w
1446 function ff_weight_h264_pixels_\w\()_neon, export=1
1447         push            {r4, lr}
1448         ldr             r12, [sp, #8]
1449         ldr             r4,  [sp, #12]
1450         cmp             r3,  #1
1451         lsl             r4,  r4,  r3
1452         vdup.16         q8,  r4
1453         mov             r4,  r0
1454         ble             20f
1455         rsb             lr,  r3,  #1
1456         vdup.16         q9,  lr
1457         cmp             r12, #0
1458         blt             10f
1459         weight_\w       vhadd.s16
1460 10:     rsb             r12, r12, #0
1461         weight_\w       vhsub.s16
1462 20:     rsb             lr,  r3,  #0
1463         vdup.16         q9,  lr
1464         cmp             r12, #0
1465         blt             10f
1466         weight_\w       vadd.s16
1467 10:     rsb             r12, r12, #0
1468         weight_\w       vsub.s16
1469 endfunc
1470 .endm
1471
1472         weight_func     16
1473         weight_func     8
1474         weight_func     4