Merge commit '21d8f4da91673051ad7055cf63d989488a56368e'
[ffmpeg.git] / libavcodec / aarch64 / h264qpel_neon.S
1 /*
2  * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
3  * Copyright (c) 2013 Janne Grunau <janne-libav@jannau.net>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 #include "libavutil/aarch64/asm.S"
23 #include "neon.S"
24
25         /* H.264 qpel MC */
26
27 .macro  lowpass_const   r
28         movz            \r, #20, lsl #16
29         movk            \r, #5
30         mov             v6.S[0], \r
31 .endm
32
33 //trashes v0-v5
34 .macro  lowpass_8       r0,  r1,  r2,  r3,  d0,  d1,  narrow=1
35         ext             v2.8B,      \r0\().8B, \r1\().8B, #2
36         ext             v3.8B,      \r0\().8B, \r1\().8B, #3
37         uaddl           v2.8H,      v2.8B,     v3.8B
38         ext             v4.8B,      \r0\().8B, \r1\().8B, #1
39         ext             v5.8B,      \r0\().8B, \r1\().8B, #4
40         uaddl           v4.8H,      v4.8B,     v5.8B
41         ext             v1.8B,      \r0\().8B, \r1\().8B, #5
42         uaddl           \d0\().8H,  \r0\().8B, v1.8B
43         ext             v0.8B,      \r2\().8B, \r3\().8B, #2
44         mla             \d0\().8H,  v2.8H,     v6.H[1]
45         ext             v1.8B,      \r2\().8B, \r3\().8B, #3
46         uaddl           v0.8H,      v0.8B,     v1.8B
47         ext             v1.8B,      \r2\().8B, \r3\().8B, #1
48         mls             \d0\().8H,  v4.8H,     v6.H[0]
49         ext             v3.8B,      \r2\().8B, \r3\().8B, #4
50         uaddl           v1.8H,      v1.8B,     v3.8B
51         ext             v2.8B,      \r2\().8B, \r3\().8B, #5
52         uaddl           \d1\().8H,  \r2\().8B, v2.8B
53         mla             \d1\().8H,  v0.8H,     v6.H[1]
54         mls             \d1\().8H,  v1.8H,     v6.H[0]
55   .if \narrow
56         sqrshrun        \d0\().8B,  \d0\().8H, #5
57         sqrshrun        \d1\().8B,  \d1\().8H, #5
58   .endif
59 .endm
60
61 //trashes v0-v5, v7, v30-v31
62 .macro  lowpass_8H      r0,  r1
63         ext             v0.16B,     \r0\().16B, \r0\().16B, #2
64         ext             v1.16B,     \r0\().16B, \r0\().16B, #3
65         uaddl           v0.8H,      v0.8B,      v1.8B
66         ext             v2.16B,     \r0\().16B, \r0\().16B, #1
67         ext             v3.16B,     \r0\().16B, \r0\().16B, #4
68         uaddl           v2.8H,      v2.8B,      v3.8B
69         ext             v30.16B,    \r0\().16B, \r0\().16B, #5
70         uaddl           \r0\().8H,  \r0\().8B,  v30.8B
71         ext             v4.16B,     \r1\().16B, \r1\().16B, #2
72         mla             \r0\().8H,  v0.8H,      v6.H[1]
73         ext             v5.16B,     \r1\().16B, \r1\().16B, #3
74         uaddl           v4.8H,      v4.8B,      v5.8B
75         ext             v7.16B,     \r1\().16B, \r1\().16B, #1
76         mls             \r0\().8H,  v2.8H,      v6.H[0]
77         ext             v0.16B,     \r1\().16B, \r1\().16B, #4
78         uaddl           v7.8H,      v7.8B,      v0.8B
79         ext             v31.16B,    \r1\().16B, \r1\().16B, #5
80         uaddl           \r1\().8H,  \r1\().8B,  v31.8B
81         mla             \r1\().8H,  v4.8H,      v6.H[1]
82         mls             \r1\().8H,  v7.8H,      v6.H[0]
83 .endm
84
85 // trashes v2-v5, v30
86 .macro  lowpass_8_1     r0,  r1,  d0,  narrow=1
87         ext             v2.8B,     \r0\().8B, \r1\().8B, #2
88         ext             v3.8B,     \r0\().8B, \r1\().8B, #3
89         uaddl           v2.8H,     v2.8B,     v3.8B
90         ext             v4.8B,     \r0\().8B, \r1\().8B, #1
91         ext             v5.8B,     \r0\().8B, \r1\().8B, #4
92         uaddl           v4.8H,     v4.8B,     v5.8B
93         ext             v30.8B,    \r0\().8B, \r1\().8B, #5
94         uaddl           \d0\().8H, \r0\().8B, v30.8B
95         mla             \d0\().8H, v2.8H,     v6.H[1]
96         mls             \d0\().8H, v4.8H,     v6.H[0]
97   .if \narrow
98         sqrshrun        \d0\().8B, \d0\().8H, #5
99   .endif
100 .endm
101
102 // trashed v0-v7
103 .macro  lowpass_8.16    r0,  r1,  r2
104         ext             v1.16B,     \r0\().16B, \r1\().16B, #4
105         ext             v0.16B,     \r0\().16B, \r1\().16B, #6
106         saddl           v5.4S,      v1.4H,      v0.4H
107         ext             v2.16B,     \r0\().16B, \r1\().16B, #2
108         saddl2          v1.4S,      v1.8H,      v0.8H
109         ext             v3.16B,     \r0\().16B, \r1\().16B, #8
110         saddl           v6.4S,      v2.4H,      v3.4H
111         ext             \r1\().16B, \r0\().16B, \r1\().16B, #10
112         saddl2          v2.4S,      v2.8H,      v3.8H
113         saddl           v0.4S,      \r0\().4H,  \r1\().4H
114         saddl2          v4.4S,      \r0\().8H,  \r1\().8H
115
116         shl             v3.4S,  v5.4S,  #4
117         shl             v5.4S,  v5.4S,  #2
118         shl             v7.4S,  v6.4S,  #2
119         add             v5.4S,  v5.4S,  v3.4S
120         add             v6.4S,  v6.4S,  v7.4S
121
122         shl             v3.4S,  v1.4S,  #4
123         shl             v1.4S,  v1.4S,  #2
124         shl             v7.4S,  v2.4S,  #2
125         add             v1.4S,  v1.4S,  v3.4S
126         add             v2.4S,  v2.4S,  v7.4S
127
128         add             v5.4S,  v5.4S,  v0.4S
129         sub             v5.4S,  v5.4S,  v6.4S
130
131         add             v1.4S,  v1.4S,  v4.4S
132         sub             v1.4S,  v1.4S,  v2.4S
133
134         rshrn           v5.4H,  v5.4S,  #10
135         rshrn2          v5.8H,  v1.4S,  #10
136
137         sqxtun          \r2\().8B,  v5.8H
138 .endm
139
140 function put_h264_qpel16_h_lowpass_neon_packed
141         mov             x4,  x30
142         mov             x12, #16
143         mov             x3,  #8
144         bl              put_h264_qpel8_h_lowpass_neon
145         sub             x1,  x1,  x2, lsl #4
146         add             x1,  x1,  #8
147         mov             x12, #16
148         mov             x30, x4
149         b               put_h264_qpel8_h_lowpass_neon
150 endfunc
151
152 .macro  h264_qpel_h_lowpass type
153 function \type\()_h264_qpel16_h_lowpass_neon
154         mov             x13, x30
155         mov             x12, #16
156         bl              \type\()_h264_qpel8_h_lowpass_neon
157         sub             x0,  x0,  x3, lsl #4
158         sub             x1,  x1,  x2, lsl #4
159         add             x0,  x0,  #8
160         add             x1,  x1,  #8
161         mov             x12, #16
162         mov             x30, x13
163 endfunc
164
165 function \type\()_h264_qpel8_h_lowpass_neon
166 1:      ld1             {v28.8B, v29.8B}, [x1], x2
167         ld1             {v16.8B, v17.8B}, [x1], x2
168         subs            x12, x12, #2
169         lowpass_8       v28, v29, v16, v17, v28, v16
170   .ifc \type,avg
171         ld1             {v2.8B},    [x0], x3
172         urhadd          v28.8B, v28.8B,  v2.8B
173         ld1             {v3.8B},    [x0]
174         urhadd          v16.8B, v16.8B, v3.8B
175         sub             x0,  x0,  x3
176   .endif
177         st1             {v28.8B},    [x0], x3
178         st1             {v16.8B},    [x0], x3
179         b.ne            1b
180         ret
181 endfunc
182 .endm
183
184         h264_qpel_h_lowpass put
185         h264_qpel_h_lowpass avg
186
187 .macro  h264_qpel_h_lowpass_l2 type
188 function \type\()_h264_qpel16_h_lowpass_l2_neon
189         mov             x13, x30
190         mov             x12, #16
191         bl              \type\()_h264_qpel8_h_lowpass_l2_neon
192         sub             x0,  x0,  x2, lsl #4
193         sub             x1,  x1,  x2, lsl #4
194         sub             x3,  x3,  x2, lsl #4
195         add             x0,  x0,  #8
196         add             x1,  x1,  #8
197         add             x3,  x3,  #8
198         mov             x12, #16
199         mov             x30, x13
200 endfunc
201
202 function \type\()_h264_qpel8_h_lowpass_l2_neon
203 1:      ld1             {v26.8B, v27.8B}, [x1], x2
204         ld1             {v16.8B, v17.8B}, [x1], x2
205         ld1             {v28.8B},     [x3], x2
206         ld1             {v29.8B},     [x3], x2
207         subs            x12, x12, #2
208         lowpass_8       v26, v27, v16, v17, v26, v27
209         urhadd          v26.8B, v26.8B, v28.8B
210         urhadd          v27.8B, v27.8B, v29.8B
211   .ifc \type,avg
212         ld1             {v2.8B},      [x0], x2
213         urhadd          v26.8B, v26.8B, v2.8B
214         ld1             {v3.8B},      [x0]
215         urhadd          v27.8B, v27.8B, v3.8B
216         sub             x0,  x0,  x2
217   .endif
218         st1             {v26.8B},     [x0], x2
219         st1             {v27.8B},     [x0], x2
220         b.ne            1b
221         ret
222 endfunc
223 .endm
224
225         h264_qpel_h_lowpass_l2 put
226         h264_qpel_h_lowpass_l2 avg
227
228 function put_h264_qpel16_v_lowpass_neon_packed
229         mov             x4,  x30
230         mov             x2,  #8
231         bl              put_h264_qpel8_v_lowpass_neon
232         sub             x1,  x1,  x3, lsl #2
233         bl              put_h264_qpel8_v_lowpass_neon
234         sub             x1,  x1,  x3, lsl #4
235         sub             x1,  x1,  x3, lsl #2
236         add             x1,  x1,  #8
237         bl              put_h264_qpel8_v_lowpass_neon
238         sub             x1,  x1,  x3, lsl #2
239         mov             x30, x4
240         b               put_h264_qpel8_v_lowpass_neon
241 endfunc
242
243 .macro  h264_qpel_v_lowpass type
244 function \type\()_h264_qpel16_v_lowpass_neon
245         mov             x4,  x30
246         bl              \type\()_h264_qpel8_v_lowpass_neon
247         sub             x1,  x1,  x3, lsl #2
248         bl              \type\()_h264_qpel8_v_lowpass_neon
249         sub             x0,  x0,  x2, lsl #4
250         add             x0,  x0,  #8
251         sub             x1,  x1,  x3, lsl #4
252         sub             x1,  x1,  x3, lsl #2
253         add             x1,  x1,  #8
254         bl              \type\()_h264_qpel8_v_lowpass_neon
255         sub             x1,  x1,  x3, lsl #2
256         mov             x30, x4
257 endfunc
258
259 function \type\()_h264_qpel8_v_lowpass_neon
260         ld1             {v16.8B}, [x1], x3
261         ld1             {v18.8B}, [x1], x3
262         ld1             {v20.8B}, [x1], x3
263         ld1             {v22.8B}, [x1], x3
264         ld1             {v24.8B}, [x1], x3
265         ld1             {v26.8B}, [x1], x3
266         ld1             {v28.8B}, [x1], x3
267         ld1             {v30.8B}, [x1], x3
268         ld1             {v17.8B}, [x1], x3
269         ld1             {v19.8B}, [x1], x3
270         ld1             {v21.8B}, [x1], x3
271         ld1             {v23.8B}, [x1], x3
272         ld1             {v25.8B}, [x1]
273
274         transpose_8x8B  v16, v18, v20, v22, v24, v26, v28, v30, v0,  v1
275         transpose_8x8B  v17, v19, v21, v23, v25, v27, v29, v31, v0,  v1
276         lowpass_8       v16, v17, v18, v19, v16, v17
277         lowpass_8       v20, v21, v22, v23, v18, v19
278         lowpass_8       v24, v25, v26, v27, v20, v21
279         lowpass_8       v28, v29, v30, v31, v22, v23
280         transpose_8x8B  v16, v17, v18, v19, v20, v21, v22, v23, v0,  v1
281
282   .ifc \type,avg
283         ld1             {v24.8B},  [x0], x2
284         urhadd          v16.8B, v16.8B, v24.8B
285         ld1             {v25.8B}, [x0], x2
286         urhadd          v17.8B, v17.8B, v25.8B
287         ld1             {v26.8B}, [x0], x2
288         urhadd          v18.8B, v18.8B, v26.8B
289         ld1             {v27.8B}, [x0], x2
290         urhadd          v19.8B, v19.8B, v27.8B
291         ld1             {v28.8B}, [x0], x2
292         urhadd          v20.8B, v20.8B, v28.8B
293         ld1             {v29.8B}, [x0], x2
294         urhadd          v21.8B, v21.8B, v29.8B
295         ld1             {v30.8B}, [x0], x2
296         urhadd          v22.8B, v22.8B, v30.8B
297         ld1             {v31.8B}, [x0], x2
298         urhadd          v23.8B, v23.8B, v31.8B
299         sub             x0,  x0,  x2,  lsl #3
300   .endif
301
302         st1             {v16.8B}, [x0], x2
303         st1             {v17.8B}, [x0], x2
304         st1             {v18.8B}, [x0], x2
305         st1             {v19.8B}, [x0], x2
306         st1             {v20.8B}, [x0], x2
307         st1             {v21.8B}, [x0], x2
308         st1             {v22.8B}, [x0], x2
309         st1             {v23.8B}, [x0], x2
310
311         ret
312 endfunc
313 .endm
314
315         h264_qpel_v_lowpass put
316         h264_qpel_v_lowpass avg
317
318 .macro  h264_qpel_v_lowpass_l2 type
319 function \type\()_h264_qpel16_v_lowpass_l2_neon
320         mov             x4,  x30
321         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
322         sub             x1,  x1,  x3, lsl #2
323         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
324         sub             x0,  x0,  x3, lsl #4
325         sub             x12, x12, x2, lsl #4
326         add             x0,  x0,  #8
327         add             x12, x12, #8
328         sub             x1,  x1,  x3, lsl #4
329         sub             x1,  x1,  x3, lsl #2
330         add             x1,  x1,  #8
331         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
332         sub             x1,  x1,  x3, lsl #2
333         mov             x30, x4
334 endfunc
335
336 function \type\()_h264_qpel8_v_lowpass_l2_neon
337         ld1             {v16.8B}, [x1], x3
338         ld1             {v18.8B}, [x1], x3
339         ld1             {v20.8B}, [x1], x3
340         ld1             {v22.8B}, [x1], x3
341         ld1             {v24.8B}, [x1], x3
342         ld1             {v26.8B}, [x1], x3
343         ld1             {v28.8B}, [x1], x3
344         ld1             {v30.8B}, [x1], x3
345         ld1             {v17.8B}, [x1], x3
346         ld1             {v19.8B}, [x1], x3
347         ld1             {v21.8B}, [x1], x3
348         ld1             {v23.8B}, [x1], x3
349         ld1             {v25.8B}, [x1]
350
351         transpose_8x8B  v16, v18, v20, v22, v24, v26, v28, v30, v0,  v1
352         transpose_8x8B  v17, v19, v21, v23, v25, v27, v29, v31, v0,  v1
353         lowpass_8       v16, v17, v18, v19, v16, v17
354         lowpass_8       v20, v21, v22, v23, v18, v19
355         lowpass_8       v24, v25, v26, v27, v20, v21
356         lowpass_8       v28, v29, v30, v31, v22, v23
357         transpose_8x8B  v16, v17, v18, v19, v20, v21, v22, v23, v0,  v1
358
359         ld1             {v24.8B},  [x12], x2
360         ld1             {v25.8B},  [x12], x2
361         ld1             {v26.8B},  [x12], x2
362         ld1             {v27.8B},  [x12], x2
363         ld1             {v28.8B},  [x12], x2
364         urhadd          v16.8B, v24.8B, v16.8B
365         urhadd          v17.8B, v25.8B, v17.8B
366         ld1             {v29.8B},  [x12], x2
367         urhadd          v18.8B, v26.8B, v18.8B
368         urhadd          v19.8B, v27.8B, v19.8B
369         ld1             {v30.8B}, [x12], x2
370         urhadd          v20.8B, v28.8B, v20.8B
371         urhadd          v21.8B, v29.8B, v21.8B
372         ld1             {v31.8B}, [x12], x2
373         urhadd          v22.8B, v30.8B, v22.8B
374         urhadd          v23.8B, v31.8B, v23.8B
375
376   .ifc \type,avg
377         ld1             {v24.8B}, [x0], x3
378         urhadd          v16.8B, v16.8B, v24.8B
379         ld1             {v25.8B}, [x0], x3
380         urhadd          v17.8B, v17.8B, v25.8B
381         ld1             {v26.8B}, [x0], x3
382         urhadd          v18.8B, v18.8B, v26.8B
383         ld1             {v27.8B}, [x0], x3
384         urhadd          v19.8B, v19.8B, v27.8B
385         ld1             {v28.8B}, [x0], x3
386         urhadd          v20.8B, v20.8B, v28.8B
387         ld1             {v29.8B}, [x0], x3
388         urhadd          v21.8B, v21.8B, v29.8B
389         ld1             {v30.8B}, [x0], x3
390         urhadd          v22.8B, v22.8B, v30.8B
391         ld1             {v31.8B}, [x0], x3
392         urhadd          v23.8B, v23.8B, v31.8B
393         sub             x0,  x0,  x3,  lsl #3
394   .endif
395
396         st1             {v16.8B}, [x0], x3
397         st1             {v17.8B}, [x0], x3
398         st1             {v18.8B}, [x0], x3
399         st1             {v19.8B}, [x0], x3
400         st1             {v20.8B}, [x0], x3
401         st1             {v21.8B}, [x0], x3
402         st1             {v22.8B}, [x0], x3
403         st1             {v23.8B}, [x0], x3
404
405         ret
406 endfunc
407 .endm
408
409         h264_qpel_v_lowpass_l2 put
410         h264_qpel_v_lowpass_l2 avg
411
412 function put_h264_qpel8_hv_lowpass_neon_top
413         lowpass_const   w12
414         ld1             {v16.8H}, [x1], x3
415         ld1             {v17.8H}, [x1], x3
416         ld1             {v18.8H}, [x1], x3
417         ld1             {v19.8H}, [x1], x3
418         ld1             {v20.8H}, [x1], x3
419         ld1             {v21.8H}, [x1], x3
420         ld1             {v22.8H}, [x1], x3
421         ld1             {v23.8H}, [x1], x3
422         ld1             {v24.8H}, [x1], x3
423         ld1             {v25.8H}, [x1], x3
424         ld1             {v26.8H}, [x1], x3
425         ld1             {v27.8H}, [x1], x3
426         ld1             {v28.8H}, [x1]
427         lowpass_8H      v16, v17
428         lowpass_8H      v18, v19
429         lowpass_8H      v20, v21
430         lowpass_8H      v22, v23
431         lowpass_8H      v24, v25
432         lowpass_8H      v26, v27
433         lowpass_8H      v28, v29
434
435         transpose_8x8H  v16, v17, v18, v19, v20, v21, v22, v23, v0,  v1
436         transpose_8x8H  v24, v25, v26, v27, v28, v29, v30, v31, v0,  v1
437
438         lowpass_8.16    v16, v24, v16
439         lowpass_8.16    v17, v25, v17
440
441         lowpass_8.16    v18, v26, v18
442         lowpass_8.16    v19, v27, v19
443
444         lowpass_8.16    v20, v28, v20
445         lowpass_8.16    v21, v29, v21
446
447         lowpass_8.16    v22, v30, v22
448         lowpass_8.16    v23, v31, v23
449
450         transpose_8x8B v16, v17, v18, v19, v20, v21, v22, v23, v0,  v1
451
452         ret
453 endfunc
454
455 .macro  h264_qpel8_hv_lowpass type
456 function \type\()_h264_qpel8_hv_lowpass_neon
457         mov             x10, x30
458         bl              put_h264_qpel8_hv_lowpass_neon_top
459   .ifc \type,avg
460         ld1             {v0.8B},      [x0], x2
461         urhadd          v16.8B, v16.8B, v0.8B
462         ld1             {v1.8B},      [x0], x2
463         urhadd          v17.8B, v17.8B, v1.8B
464         ld1             {v2.8B},      [x0], x2
465         urhadd          v18.8B, v18.8B, v2.8B
466         ld1             {v3.8B},      [x0], x2
467         urhadd          v19.8B, v19.8B, v3.8B
468         ld1             {v4.8B},      [x0], x2
469         urhadd          v20.8B, v20.8B, v4.8B
470         ld1             {v5.8B},      [x0], x2
471         urhadd          v21.8B, v21.8B, v5.8B
472         ld1             {v6.8B},      [x0], x2
473         urhadd          v22.8B, v22.8B, v6.8B
474         ld1             {v7.8B},      [x0], x2
475         urhadd          v23.8B, v23.8B, v7.8B
476         sub             x0,  x0,  x2,  lsl #3
477   .endif
478
479         st1             {v16.8B},     [x0], x2
480         st1             {v17.8B},     [x0], x2
481         st1             {v18.8B},     [x0], x2
482         st1             {v19.8B},     [x0], x2
483         st1             {v20.8B},     [x0], x2
484         st1             {v21.8B},     [x0], x2
485         st1             {v22.8B},     [x0], x2
486         st1             {v23.8B},     [x0], x2
487
488         ret             x10
489 endfunc
490 .endm
491
492         h264_qpel8_hv_lowpass put
493         h264_qpel8_hv_lowpass avg
494
495 .macro  h264_qpel8_hv_lowpass_l2 type
496 function \type\()_h264_qpel8_hv_lowpass_l2_neon
497         mov             x10, x30
498         bl              put_h264_qpel8_hv_lowpass_neon_top
499
500         ld1             {v0.8B, v1.8B},  [x2], #16
501         ld1             {v2.8B, v3.8B},  [x2], #16
502         urhadd          v0.8B,  v0.8B,  v16.8B
503         urhadd          v1.8B,  v1.8B,  v17.8B
504         ld1             {v4.8B, v5.8B},  [x2], #16
505         urhadd          v2.8B,  v2.8B,  v18.8B
506         urhadd          v3.8B,  v3.8B,  v19.8B
507         ld1             {v6.8B, v7.8B},  [x2], #16
508         urhadd          v4.8B,  v4.8B,  v20.8B
509         urhadd          v5.8B,  v5.8B,  v21.8B
510         urhadd          v6.8B,  v6.8B,  v22.8B
511         urhadd          v7.8B,  v7.8B,  v23.8B
512   .ifc \type,avg
513         ld1             {v16.8B},     [x0], x3
514         urhadd          v0.8B,  v0.8B,  v16.8B
515         ld1             {v17.8B},     [x0], x3
516         urhadd          v1.8B,  v1.8B,  v17.8B
517         ld1             {v18.8B},     [x0], x3
518         urhadd          v2.8B,  v2.8B,  v18.8B
519         ld1             {v19.8B},     [x0], x3
520         urhadd          v3.8B,  v3.8B,  v19.8B
521         ld1             {v20.8B},     [x0], x3
522         urhadd          v4.8B,  v4.8B,  v20.8B
523         ld1             {v21.8B},     [x0], x3
524         urhadd          v5.8B,  v5.8B,  v21.8B
525         ld1             {v22.8B},     [x0], x3
526         urhadd          v6.8B,  v6.8B,  v22.8B
527         ld1             {v23.8B},     [x0], x3
528         urhadd          v7.8B,  v7.8B,  v23.8B
529         sub             x0,  x0,  x3,  lsl #3
530   .endif
531         st1             {v0.8B},      [x0], x3
532         st1             {v1.8B},      [x0], x3
533         st1             {v2.8B},      [x0], x3
534         st1             {v3.8B},      [x0], x3
535         st1             {v4.8B},      [x0], x3
536         st1             {v5.8B},      [x0], x3
537         st1             {v6.8B},      [x0], x3
538         st1             {v7.8B},      [x0], x3
539
540         ret             x10
541 endfunc
542 .endm
543
544         h264_qpel8_hv_lowpass_l2 put
545         h264_qpel8_hv_lowpass_l2 avg
546
547 .macro  h264_qpel16_hv  type
548 function \type\()_h264_qpel16_hv_lowpass_neon
549         mov             x13, x30
550         bl              \type\()_h264_qpel8_hv_lowpass_neon
551         sub             x1,  x1,  x3, lsl #2
552         bl              \type\()_h264_qpel8_hv_lowpass_neon
553         sub             x1,  x1,  x3, lsl #4
554         sub             x1,  x1,  x3, lsl #2
555         add             x1,  x1,  #8
556         sub             x0,  x0,  x2, lsl #4
557         add             x0,  x0,  #8
558         bl              \type\()_h264_qpel8_hv_lowpass_neon
559         sub             x1,  x1,  x3, lsl #2
560         mov             x30, x13
561         b               \type\()_h264_qpel8_hv_lowpass_neon
562 endfunc
563
564 function \type\()_h264_qpel16_hv_lowpass_l2_neon
565         mov             x13, x30
566         sub             x2,  x4,  #256
567         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
568         sub             x1,  x1,  x3, lsl #2
569         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
570         sub             x1,  x1,  x3, lsl #4
571         sub             x1,  x1,  x3, lsl #2
572         add             x1,  x1,  #8
573         sub             x0,  x0,  x3, lsl #4
574         add             x0,  x0,  #8
575         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
576         sub             x1,  x1,  x3, lsl #2
577         mov             x30, x13
578         b               \type\()_h264_qpel8_hv_lowpass_l2_neon
579 endfunc
580 .endm
581
582         h264_qpel16_hv put
583         h264_qpel16_hv avg
584
585 .macro  h264_qpel8      type
586 function ff_\type\()_h264_qpel8_mc10_neon, export=1
587         lowpass_const   w3
588         mov             x3,  x1
589         sub             x1,  x1,  #2
590         mov             x12, #8
591         b               \type\()_h264_qpel8_h_lowpass_l2_neon
592 endfunc
593
594 function ff_\type\()_h264_qpel8_mc20_neon, export=1
595         lowpass_const   w3
596         sub             x1,  x1,  #2
597         mov             x3,  x2
598         mov             x12, #8
599         b               \type\()_h264_qpel8_h_lowpass_neon
600 endfunc
601
602 function ff_\type\()_h264_qpel8_mc30_neon, export=1
603         lowpass_const   w3
604         add             x3,  x1,  #1
605         sub             x1,  x1,  #2
606         mov             x12, #8
607         b               \type\()_h264_qpel8_h_lowpass_l2_neon
608 endfunc
609
610 function ff_\type\()_h264_qpel8_mc01_neon, export=1
611         mov             x14, x30
612         mov             x12, x1
613 \type\()_h264_qpel8_mc01:
614         lowpass_const   w3
615         mov             x3,  x2
616         sub             x1,  x1,  x2, lsl #1
617         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
618         ret             x14
619 endfunc
620
621 function ff_\type\()_h264_qpel8_mc11_neon, export=1
622         mov             x14, x30
623         mov             x8,  x0
624         mov             x9,  x1
625 \type\()_h264_qpel8_mc11:
626         lowpass_const   w3
627         mov             x11, sp
628         sub             sp,  sp,  #64
629         mov             x0,  sp
630         sub             x1,  x1,  #2
631         mov             x3,  #8
632         mov             x12, #8
633         bl              put_h264_qpel8_h_lowpass_neon
634         mov             x0,  x8
635         mov             x3,  x2
636         mov             x12, sp
637         sub             x1,  x9,  x2, lsl #1
638         mov             x2,  #8
639         bl              \type\()_h264_qpel8_v_lowpass_l2_neon
640         mov             sp,  x11
641         ret             x14
642 endfunc
643
644 function ff_\type\()_h264_qpel8_mc21_neon, export=1
645         mov             x14, x30
646         mov             x8,  x0
647         mov             x9,  x1
648 \type\()_h264_qpel8_mc21:
649         lowpass_const   w3
650         mov             x11, sp
651         sub             sp,  sp,  #(8*8+16*12)
652         sub             x1,  x1,  #2
653         mov             x3,  #8
654         mov             x0,  sp
655         mov             x12, #8
656         bl              put_h264_qpel8_h_lowpass_neon
657         mov             x4,  x0
658         mov             x0,  x8
659         sub             x1,  x9,  x2, lsl #1
660         sub             x1,  x1,  #2
661         mov             x3,  x2
662         sub             x2,  x4,  #64
663         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
664         mov             sp,  x11
665         ret             x14
666 endfunc
667
668 function ff_\type\()_h264_qpel8_mc31_neon, export=1
669         add             x1,  x1,  #1
670         mov             x14, x30
671         mov             x8,  x0
672         mov             x9,  x1
673         sub             x1,  x1,  #1
674         b               \type\()_h264_qpel8_mc11
675 endfunc
676
677 function ff_\type\()_h264_qpel8_mc02_neon, export=1
678         mov             x14, x30
679         lowpass_const   w3
680         sub             x1,  x1,  x2, lsl #1
681         mov             x3,  x2
682         bl              \type\()_h264_qpel8_v_lowpass_neon
683         ret             x14
684 endfunc
685
686 function ff_\type\()_h264_qpel8_mc12_neon, export=1
687         mov             x14, x30
688         mov             x8,  x0
689         mov             x9,  x1
690 \type\()_h264_qpel8_mc12:
691         lowpass_const   w3
692         mov             x11, sp
693         sub             sp,  sp,  #(8*8+16*12)
694         sub             x1,  x1,  x2, lsl #1
695         mov             x3,  x2
696         mov             x2,  #8
697         mov             x0,  sp
698         bl              put_h264_qpel8_v_lowpass_neon
699         mov             x4,  x0
700         mov             x0,  x8
701         sub             x1,  x9,  x3, lsl #1
702         sub             x1,  x1,  #2
703         sub             x2,  x4,  #64
704         bl              \type\()_h264_qpel8_hv_lowpass_l2_neon
705         mov             sp,  x11
706         ret             x14
707 endfunc
708
709 function ff_\type\()_h264_qpel8_mc22_neon, export=1
710         mov             x14, x30
711         mov             x11, sp
712         sub             x1,  x1,  x2, lsl #1
713         sub             x1,  x1,  #2
714         mov             x3,  x2
715         bl              \type\()_h264_qpel8_hv_lowpass_neon
716         mov             sp,  x11
717         ret             x14
718 endfunc
719
720 function ff_\type\()_h264_qpel8_mc32_neon, export=1
721         mov             x14, x30
722         mov             x8,  x0
723         mov             x9,  x1
724         add             x1,  x1,  #1
725         b               \type\()_h264_qpel8_mc12
726 endfunc
727
728 function ff_\type\()_h264_qpel8_mc03_neon, export=1
729         mov             x14, x30
730         add             x12, x1,  x2
731         b               \type\()_h264_qpel8_mc01
732 endfunc
733
734 function ff_\type\()_h264_qpel8_mc13_neon, export=1
735         mov             x14, x30
736         mov             x8,  x0
737         mov             x9,  x1
738         add             x1,  x1,  x2
739         b               \type\()_h264_qpel8_mc11
740 endfunc
741
742 function ff_\type\()_h264_qpel8_mc23_neon, export=1
743         mov             x14, x30
744         mov             x8,  x0
745         mov             x9,  x1
746         add             x1,  x1,  x2
747         b               \type\()_h264_qpel8_mc21
748 endfunc
749
750 function ff_\type\()_h264_qpel8_mc33_neon, export=1
751         add             x1,  x1,  #1
752         mov             x14, x30
753         mov             x8,  x0
754         mov             x9,  x1
755         add             x1,  x1,  x2
756         sub             x1,  x1,  #1
757         b               \type\()_h264_qpel8_mc11
758 endfunc
759 .endm
760
761         h264_qpel8 put
762         h264_qpel8 avg
763
764 .macro  h264_qpel16     type
765 function ff_\type\()_h264_qpel16_mc10_neon, export=1
766         lowpass_const   w3
767         mov             x3,  x1
768         sub             x1,  x1,  #2
769         b               \type\()_h264_qpel16_h_lowpass_l2_neon
770 endfunc
771
772 function ff_\type\()_h264_qpel16_mc20_neon, export=1
773         lowpass_const   w3
774         sub             x1,  x1,  #2
775         mov             x3,  x2
776         b               \type\()_h264_qpel16_h_lowpass_neon
777 endfunc
778
779 function ff_\type\()_h264_qpel16_mc30_neon, export=1
780         lowpass_const   w3
781         add             x3,  x1,  #1
782         sub             x1,  x1,  #2
783         b               \type\()_h264_qpel16_h_lowpass_l2_neon
784 endfunc
785
786 function ff_\type\()_h264_qpel16_mc01_neon, export=1
787         mov             x14, x30
788         mov             x12, x1
789 \type\()_h264_qpel16_mc01:
790         lowpass_const   w3
791         mov             x3,  x2
792         sub             x1,  x1,  x2, lsl #1
793         bl              \type\()_h264_qpel16_v_lowpass_l2_neon
794         ret             x14
795 endfunc
796
797 function ff_\type\()_h264_qpel16_mc11_neon, export=1
798         mov             x14, x30
799         mov             x8,  x0
800         mov             x9,  x1
801 \type\()_h264_qpel16_mc11:
802         lowpass_const   w3
803         mov             x11, sp
804         sub             sp,  sp,  #256
805         mov             x0,  sp
806         sub             x1,  x1,  #2
807         mov             x3,  #16
808         bl              put_h264_qpel16_h_lowpass_neon
809         mov             x0,  x8
810         mov             x3,  x2
811         mov             x12, sp
812         sub             x1,  x9,  x2, lsl #1
813         mov             x2,  #16
814         bl              \type\()_h264_qpel16_v_lowpass_l2_neon
815         mov             sp,  x11
816         ret             x14
817 endfunc
818
819 function ff_\type\()_h264_qpel16_mc21_neon, export=1
820         mov             x14, x30
821         mov             x8,  x0
822         mov             x9,  x1
823 \type\()_h264_qpel16_mc21:
824         lowpass_const   w3
825         mov             x11, sp
826         sub             sp,  sp,  #(16*16+16*12)
827         sub             x1,  x1,  #2
828         mov             x0,  sp
829         bl              put_h264_qpel16_h_lowpass_neon_packed
830         mov             x4,  x0
831         mov             x0,  x8
832         sub             x1,  x9,  x2, lsl #1
833         sub             x1,  x1,  #2
834         mov             x3,  x2
835         bl              \type\()_h264_qpel16_hv_lowpass_l2_neon
836         mov             sp,  x11
837         ret             x14
838 endfunc
839
840 function ff_\type\()_h264_qpel16_mc31_neon, export=1
841         add             x1,  x1,  #1
842         mov             x14, x30
843         mov             x8,  x0
844         mov             x9,  x1
845         sub             x1,  x1,  #1
846         b               \type\()_h264_qpel16_mc11
847 endfunc
848
849 function ff_\type\()_h264_qpel16_mc02_neon, export=1
850         mov             x14, x30
851         lowpass_const   w3
852         sub             x1,  x1,  x2, lsl #1
853         mov             x3,  x2
854         bl              \type\()_h264_qpel16_v_lowpass_neon
855         ret             x14
856 endfunc
857
858 function ff_\type\()_h264_qpel16_mc12_neon, export=1
859         mov             x14, x30
860         mov             x8,  x0
861         mov             x9,  x1
862 \type\()_h264_qpel16_mc12:
863         lowpass_const   w3
864         mov             x11, sp
865         sub             sp,  sp,  #(16*16+16*12)
866         sub             x1,  x1,  x2, lsl #1
867         mov             x0,  sp
868         mov             x3,  x2
869         bl              put_h264_qpel16_v_lowpass_neon_packed
870         mov             x4,  x0
871         mov             x0,  x8
872         sub             x1,  x9,  x3, lsl #1
873         sub             x1,  x1,  #2
874         mov             x2,  x3
875         bl              \type\()_h264_qpel16_hv_lowpass_l2_neon
876         mov             sp,  x11
877         ret             x14
878 endfunc
879
880 function ff_\type\()_h264_qpel16_mc22_neon, export=1
881         mov             x14, x30
882         lowpass_const   w3
883         mov             x11, sp
884         sub             x1,  x1,  x2, lsl #1
885         sub             x1,  x1,  #2
886         mov             x3,  x2
887         bl              \type\()_h264_qpel16_hv_lowpass_neon
888         mov             sp,  x11 // restore stack
889         ret             x14
890 endfunc
891
892 function ff_\type\()_h264_qpel16_mc32_neon, export=1
893         mov             x14, x30
894         mov             x8,  x0
895         mov             x9,  x1
896         add             x1,  x1,  #1
897         b               \type\()_h264_qpel16_mc12
898 endfunc
899
900 function ff_\type\()_h264_qpel16_mc03_neon, export=1
901         mov             x14, x30
902         add             x12, x1,  x2
903         b               \type\()_h264_qpel16_mc01
904 endfunc
905
906 function ff_\type\()_h264_qpel16_mc13_neon, export=1
907         mov             x14, x30
908         mov             x8,  x0
909         mov             x9,  x1
910         add             x1,  x1,  x2
911         b               \type\()_h264_qpel16_mc11
912 endfunc
913
914 function ff_\type\()_h264_qpel16_mc23_neon, export=1
915         mov             x14, x30
916         mov             x8,  x0
917         mov             x9,  x1
918         add             x1,  x1,  x2
919         b               \type\()_h264_qpel16_mc21
920 endfunc
921
922 function ff_\type\()_h264_qpel16_mc33_neon, export=1
923         add             x1,  x1,  #1
924         mov             x14, x30
925         mov             x8,  x0
926         mov             x9,  x1
927         add             x1,  x1,  x2
928         sub             x1,  x1,  #1
929         b               \type\()_h264_qpel16_mc11
930 endfunc
931 .endm
932
933         h264_qpel16 put
934         h264_qpel16 avg