rv40: NEON optimised rv40 qpel motion compensation
[ffmpeg.git] / libavcodec / arm / rv40dsp_neon.S
1 /*
2  * Copyright (c) 2011 Janne Grunau <janne-libav@jannau.net>
3  * Copyright (c) 2011 Mans Rullgard <mans@mansr.com>
4  *
5  * This file is part of Libav.
6  *
7  * Libav is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * Libav is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with Libav; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 #include "asm.S"
23 #include "neon.S"
24
25 .macro  qpel_lowpass    r0,  r1,  rc1, rc2, shift
26         vext.8          d25, \r0, \r1, #1       @ src[-1]
27         vext.8          d26, \r0, \r1, #4       @ src[ 2]
28         vext.8          d24, \r0, \r1, #5       @ src[ 3]
29         vaddl.u8        q9,  d25, d26
30         vaddl.u8        q8,  \r0, d24
31         vext.8          d27, \r0, \r1, #2       @ src[ 0]
32         vshl.s16        q12, q9,  #2
33         vsub.s16        q8,  q8,  q9
34         vext.8          d28, \r0, \r1, #3       @ src[ 1]
35         vsub.s16        q8,  q8,  q12
36         vmlal.u8        q8,  d27, \rc1
37         vmlal.u8        q8,  d28, \rc2
38         vqrshrun.s16    \r0, q8,  #\shift
39 .endm
40
41 .macro  qpel_lowpass_x2 r0,  r1,  r2,  r3,  rc1, rc2, shift
42         vext.8          d25, \r0, \r1, #1       @ src[-1]
43         vext.8          d26, \r0, \r1, #4       @ src[ 2]
44         vext.8          d24, \r0, \r1, #5       @ src[ 3]
45         vaddl.u8        q9,  d25, d26
46         vaddl.u8        q8,  \r0, d24
47         vext.8          d29, \r0, \r1, #2       @ src[ 0]
48         vext.8          d28, \r0, \r1, #3       @ src[ 1]
49         vshl.s16        q10, q9,  #2
50         vext.8          \r1, \r2, \r3, #1       @ src[-1]
51         vsub.s16        q8,  q8,  q9
52         vext.8          d22, \r2, \r3, #4       @ src[ 2]
53         vext.8          \r0, \r2, \r3, #5       @ src[ 3]
54         vaddl.u8        q13, \r1, d22
55         vaddl.u8        q12, \r2, \r0
56         vsub.s16        q8,  q8,  q10
57         vshl.s16        q9,  q13, #2
58         vsub.s16        q12, q12, q13
59         vmlal.u8        q8,  d29, \rc1
60         vmlal.u8        q8,  d28, \rc2
61         vsub.s16        q12, q12, q9
62         vext.8          d26, \r2, \r3, #2       @ src[ 0]
63         vext.8          d27, \r2, \r3, #3       @ src[ 1]
64         vmlal.u8        q12, d26, \rc1
65         vmlal.u8        q12, d27, \rc2
66         vqrshrun.s16    \r0, q8,  #\shift
67         vqrshrun.s16    \r2, q12, #\shift
68 .endm
69
70 .macro  rv40_qpel8_h    shift
71 function put_rv40_qpel8_h_lp_packed_s\shift\()_neon
72 1:
73         vld1.8          {q2},     [r1], r2
74         vld1.8          {q3},     [r1], r2
75         qpel_lowpass_x2 d4,  d5,  d6,  d7,  d0,  d1,  \shift
76         vst1.8          {d4},     [r12,:64]!
77         vst1.8          {d6},     [r12,:64]!
78         subs            r3,  r3,  #2
79         bgt             1b
80         vld1.8          {q2},     [r1]
81         qpel_lowpass    d4,  d5,  d0,  d1,  \shift
82         vst1.8          {d4},     [r12,:64]!
83         bx              lr
84 endfunc
85 .endm
86
87 .macro  rv40_qpel8_v    shift, type
88 function \type\()_rv40_qpel8_v_lp_packed_s\shift\()_neon
89         vld1.64         {d2},     [r1,:64]!
90         vld1.64         {d3},     [r1,:64]!
91         vld1.64         {d4},     [r1,:64]!
92         vld1.64         {d5},     [r1,:64]!
93         vld1.64         {d6},     [r1,:64]!
94         vld1.64         {d7},     [r1,:64]!
95         vld1.64         {d8},     [r1,:64]!
96         vld1.64         {d9},     [r1,:64]!
97         vld1.64         {d10},    [r1,:64]!
98         vld1.64         {d11},    [r1,:64]!
99         vld1.64         {d12},    [r1,:64]!
100         vld1.64         {d13},    [r1,:64]!
101         vld1.64         {d14},    [r1,:64]!
102         transpose_8x8   d2,  d3,  d4,  d5,  d6,  d7,  d8,  d9
103         transpose_8x8   d10, d11, d12, d13, d14, d15, d30, d31
104         qpel_lowpass_x2 d2,  d10, d3,  d11, d0,  d1,  \shift
105         qpel_lowpass_x2 d4,  d12, d5,  d13, d0,  d1,  \shift
106         qpel_lowpass_x2 d6,  d14, d7,  d15, d0,  d1,  \shift
107         qpel_lowpass_x2 d8,  d30, d9,  d31, d0,  d1,  \shift
108         transpose_8x8   d2,  d3,  d4,  d5,  d6,  d7,  d8,  d9
109   .ifc \type,avg
110         vld1.64         d12,      [r0,:64], r2
111         vld1.64         d13,      [r0,:64], r2
112         vld1.64         d14,      [r0,:64], r2
113         vld1.64         d15,      [r0,:64], r2
114         vld1.64         d16,      [r0,:64], r2
115         vld1.64         d17,      [r0,:64], r2
116         vld1.64         d18,      [r0,:64], r2
117         vld1.64         d19,      [r0,:64], r2
118         sub             r0,  r0,  r2,  lsl #3
119         vrhadd.u8       q1,  q1,  q6
120         vrhadd.u8       q2,  q2,  q7
121         vrhadd.u8       q3,  q3,  q8
122         vrhadd.u8       q4,  q4,  q9
123   .endif
124         vst1.64         d2,       [r0,:64], r2
125         vst1.64         d3,       [r0,:64], r2
126         vst1.64         d4,       [r0,:64], r2
127         vst1.64         d5,       [r0,:64], r2
128         vst1.64         d6,       [r0,:64], r2
129         vst1.64         d7,       [r0,:64], r2
130         vst1.64         d8,       [r0,:64], r2
131         vst1.64         d9,       [r0,:64], r2
132         bx              lr
133 endfunc
134 .endm
135
136         rv40_qpel8_h    5
137         rv40_qpel8_h    6
138
139 .macro  rv40_qpel       type
140 function \type\()_rv40_qpel8_h_lowpass_neon
141   .ifc \type,avg
142         mov             r12, r0
143   .endif
144 1:
145         vld1.8          {q2},     [r1], r2
146         vld1.8          {q3},     [r1], r2
147         qpel_lowpass_x2 d4,  d5,  d6,  d7,  d0,  d1,  6
148   .ifc \type,avg
149         vld1.8          {d3},     [r12,:64], r2
150         vld1.8          {d16},    [r12,:64], r2
151         vrhadd.u8       d4,  d4,  d3
152         vrhadd.u8       d6,  d6,  d16
153   .endif
154         vst1.8          {d4},     [r0,:64], r2
155         vst1.8          {d6},     [r0,:64], r2
156         subs            r3,  r3,  #2
157         bgt             1b
158         bx              lr
159 endfunc
160
161 function \type\()_rv40_qpel8_v_lowpass_neon
162         vld1.64         {d2},     [r1], r2
163         vld1.64         {d3},     [r1], r2
164         vld1.64         {d4},     [r1], r2
165         vld1.64         {d5},     [r1], r2
166         vld1.64         {d6},     [r1], r2
167         vld1.64         {d7},     [r1], r2
168         vld1.64         {d8},     [r1], r2
169         vld1.64         {d9},     [r1], r2
170         vld1.64         {d10},    [r1], r2
171         vld1.64         {d11},    [r1], r2
172         vld1.64         {d12},    [r1], r2
173         vld1.64         {d13},    [r1], r2
174         vld1.64         {d14},    [r1]
175         transpose_8x8   d2,  d3,  d4,  d5,  d6,  d7,  d8,  d9
176         transpose_8x8   d10, d11, d12, d13, d14, d15, d30, d31
177         qpel_lowpass_x2 d2,  d10, d3,  d11, d0,  d1,  6
178         qpel_lowpass_x2 d4,  d12, d5,  d13, d0,  d1,  6
179         qpel_lowpass_x2 d6,  d14, d7,  d15, d0,  d1,  6
180         qpel_lowpass_x2 d8,  d30, d9,  d31, d0,  d1,  6
181         transpose_8x8   d2,  d3,  d4,  d5,  d6,  d7,  d8,  d9
182   .ifc \type,avg
183         vld1.64         d12,      [r0,:64], r2
184         vld1.64         d13,      [r0,:64], r2
185         vld1.64         d14,      [r0,:64], r2
186         vld1.64         d15,      [r0,:64], r2
187         vld1.64         d16,      [r0,:64], r2
188         vld1.64         d17,      [r0,:64], r2
189         vld1.64         d18,      [r0,:64], r2
190         vld1.64         d19,      [r0,:64], r2
191         sub             r0,  r0,  r2,  lsl #3
192         vrhadd.u8       q1,  q1,  q6
193         vrhadd.u8       q2,  q2,  q7
194         vrhadd.u8       q3,  q3,  q8
195         vrhadd.u8       q4,  q4,  q9
196   .endif
197         vst1.64         d2,       [r0,:64], r2
198         vst1.64         d3,       [r0,:64], r2
199         vst1.64         d4,       [r0,:64], r2
200         vst1.64         d5,       [r0,:64], r2
201         vst1.64         d6,       [r0,:64], r2
202         vst1.64         d7,       [r0,:64], r2
203         vst1.64         d8,       [r0,:64], r2
204         vst1.64         d9,       [r0,:64], r2
205         bx              lr
206 endfunc
207
208         rv40_qpel8_v    5, \type
209         rv40_qpel8_v    6, \type
210
211 function ff_\type\()_rv40_qpel8_mc10_neon, export=1
212         sub             r1,  r1,  #2
213         mov             r3,  #8
214         vmov.i8         d0,  #52
215         vmov.i8         d1,  #20
216         b               \type\()_rv40_qpel8_h_lowpass_neon
217 endfunc
218
219 function ff_\type\()_rv40_qpel8_mc30_neon, export=1
220         sub             r1,  r1,  #2
221         mov             r3,  #8
222         vmov.i8         d0,  #20
223         vmov.i8         d1,  #52
224         b               \type\()_rv40_qpel8_h_lowpass_neon
225 endfunc
226
227 function ff_\type\()_rv40_qpel8_mc01_neon, export=1
228         push            {r4, lr}
229         vpush           {d8-d15}
230         sub             r1,  r1,  r2,  lsl #1
231         vmov.i8         d0,  #52
232         vmov.i8         d1,  #20
233         bl              \type\()_rv40_qpel8_v_lowpass_neon
234         vpop            {d8-d15}
235         pop             {r4, pc}
236 endfunc
237
238 function ff_\type\()_rv40_qpel8_mc11_neon, export=1
239         push            {r4, lr}
240         vpush           {d8-d15}
241         sub             sp,  sp,  #14*8
242         add             r12, sp,  #7
243         bic             r12, r12, #7
244         sub             r1,  r1,  r2,  lsl #1
245         sub             r1,  r1,  #2
246         mov             r3,  #12
247         vmov.i8         d0,  #52
248         vmov.i8         d1,  #20
249         bl              put_rv40_qpel8_h_lp_packed_s6_neon
250         add             r1,  sp,  #7
251         bic             r1,  r1,  #7
252         bl              \type\()_rv40_qpel8_v_lp_packed_s6_neon
253         add             sp,  sp,  #14*8
254         vpop            {d8-d15}
255         pop             {r4, pc}
256 endfunc
257
258 function ff_\type\()_rv40_qpel8_mc21_neon, export=1
259         push            {r4, lr}
260         vpush           {d8-d15}
261         sub             sp,  sp,  #14*8
262         add             r12, sp,  #7
263         bic             r12, r12, #7
264         sub             r1,  r1,  r2,  lsl #1
265         sub             r1,  r1,  #2
266         mov             r3,  #12
267         vmov.i8         d0,  #20
268         vmov.i8         d1,  #20
269         bl              put_rv40_qpel8_h_lp_packed_s5_neon
270         add             r1,  sp,  #7
271         bic             r1,  r1,  #7
272         vmov.i8         d0,  #52
273         bl              \type\()_rv40_qpel8_v_lp_packed_s6_neon
274         add             sp,  sp,  #14*8
275         vpop            {d8-d15}
276         pop             {r4, pc}
277 endfunc
278
279 function ff_\type\()_rv40_qpel8_mc31_neon, export=1
280         push            {r4, lr}
281         vpush           {d8-d15}
282         sub             sp,  sp,  #14*8
283         add             r12, sp,  #7
284         bic             r12, r12, #7
285         sub             r1,  r1,  r2,  lsl #1
286         sub             r1,  r1,  #2
287         mov             r3,  #12
288         vmov.i8         d0,  #20
289         vmov.i8         d1,  #52
290         bl              put_rv40_qpel8_h_lp_packed_s6_neon
291         add             r1,  sp,  #7
292         bic             r1,  r1,  #7
293         vswp            d0,  d1
294         bl              \type\()_rv40_qpel8_v_lp_packed_s6_neon
295         add             sp,  sp,  #14*8
296         vpop            {d8-d15}
297         pop             {r4, pc}
298 endfunc
299
300 function ff_\type\()_rv40_qpel8_mc12_neon, export=1
301         push            {r4, lr}
302         vpush           {d8-d15}
303         sub             sp,  sp,  #14*8
304         add             r12, sp,  #7
305         bic             r12, r12, #7
306         sub             r1,  r1,  r2,  lsl #1
307         sub             r1,  r1,  #2
308         mov             r3,  #12
309         vmov.i8         d0,  #52
310         vmov.i8         d1,  #20
311         bl              put_rv40_qpel8_h_lp_packed_s6_neon
312         add             r1,  sp,  #7
313         bic             r1,  r1,  #7
314         vmov.i8         d0,  #20
315         bl              \type\()_rv40_qpel8_v_lp_packed_s5_neon
316         add             sp,  sp,  #14*8
317         vpop            {d8-d15}
318         pop             {r4, pc}
319 endfunc
320
321 function ff_\type\()_rv40_qpel8_mc22_neon, export=1
322         push            {r4, lr}
323         vpush           {d8-d15}
324         sub             sp,  sp,  #14*8
325         add             r12, sp,  #7
326         bic             r12, r12, #7
327         sub             r1,  r1,  r2,  lsl #1
328         sub             r1,  r1,  #2
329         mov             r3,  #12
330         vmov.i8         d0,  #20
331         vmov.i8         d1,  #20
332         bl              put_rv40_qpel8_h_lp_packed_s5_neon
333         add             r1,  sp,  #7
334         bic             r1,  r1,  #7
335         bl              \type\()_rv40_qpel8_v_lp_packed_s5_neon
336         add             sp,  sp,  #14*8
337         vpop            {d8-d15}
338         pop             {r4, pc}
339 endfunc
340
341 function ff_\type\()_rv40_qpel8_mc32_neon, export=1
342         push            {r4, lr}
343         vpush           {d8-d15}
344         sub             sp,  sp,  #14*8
345         add             r12, sp,  #7
346         bic             r12, r12, #7
347         sub             r1,  r1,  r2,  lsl #1
348         sub             r1,  r1,  #2
349         mov             r3,  #12
350         vmov.i8         d0,  #20
351         vmov.i8         d1,  #52
352         bl              put_rv40_qpel8_h_lp_packed_s6_neon
353         add             r1,  sp,  #7
354         bic             r1,  r1,  #7
355         vmov.i8         d1,  #20
356         bl              \type\()_rv40_qpel8_v_lp_packed_s5_neon
357         add             sp,  sp,  #14*8
358         vpop            {d8-d15}
359         pop             {r4, pc}
360 endfunc
361
362 function ff_\type\()_rv40_qpel8_mc03_neon, export=1
363         push            {r4, lr}
364         vpush           {d8-d15}
365         sub             r1,  r1,  r2,  lsl #1
366         vmov.i8         d0,  #20
367         vmov.i8         d1,  #52
368         bl              \type\()_rv40_qpel8_v_lowpass_neon
369         vpop            {d8-d15}
370         pop             {r4, pc}
371 endfunc
372
373 function ff_\type\()_rv40_qpel8_mc33_neon, export=1
374         mov             r3,  #8
375         b               ff_\type\()_pixels8_xy2_neon
376 endfunc
377
378 function ff_\type\()_rv40_qpel8_mc13_neon, export=1
379         push            {r4, lr}
380         vpush           {d8-d15}
381         sub             sp,  sp,  #14*8
382         add             r12, sp,  #7
383         bic             r12, r12, #7
384         sub             r1,  r1,  r2,  lsl #1
385         sub             r1,  r1,  #2
386         mov             r3,  #12
387         vmov.i8         d0,  #52
388         vmov.i8         d1,  #20
389         bl              put_rv40_qpel8_h_lp_packed_s6_neon
390         add             r1,  sp,  #7
391         bic             r1,  r1,  #7
392         vswp            d0,  d1
393         bl              \type\()_rv40_qpel8_v_lp_packed_s6_neon
394         add             sp,  sp,  #14*8
395         vpop            {d8-d15}
396         pop             {r4, pc}
397 endfunc
398
399 function ff_\type\()_rv40_qpel8_mc23_neon, export=1
400         push            {r4, lr}
401         vpush           {d8-d15}
402         sub             sp,  sp,  #14*8
403         add             r12, sp,  #7
404         bic             r12, r12, #7
405         sub             r1,  r1,  r2,  lsl #1
406         sub             r1,  r1,  #2
407         mov             r3,  #12
408         vmov.i8         d0,  #20
409         vmov.i8         d1,  #20
410         bl              put_rv40_qpel8_h_lp_packed_s5_neon
411         add             r1,  sp,  #7
412         bic             r1,  r1,  #7
413         vmov.i8         d1,  #52
414         bl              \type\()_rv40_qpel8_v_lp_packed_s6_neon
415         add             sp,  sp,  #14*8
416         vpop            {d8-d15}
417         pop             {r4, pc}
418 endfunc
419
420 function ff_\type\()_rv40_qpel16_mc10_neon, export=1
421         vmov.i8         d0,  #52
422         vmov.i8         d1,  #20
423 .L\type\()_rv40_qpel16_h:
424         push            {r1, lr}
425         sub             r1,  r1,  #2
426         mov             r3,  #16
427         bl              \type\()_rv40_qpel8_h_lowpass_neon
428         pop             {r1, lr}
429         sub             r0,  r0,  r2,  lsl #4
430         add             r0,  r0,  #8
431         add             r1,  r1,  #6
432         mov             r3,  #16
433         b               \type\()_rv40_qpel8_h_lowpass_neon
434 endfunc
435
436 function ff_\type\()_rv40_qpel16_mc30_neon, export=1
437         vmov.i8         d0,  #20
438         vmov.i8         d1,  #52
439         b               .L\type\()_rv40_qpel16_h
440 endfunc
441
442 function ff_\type\()_rv40_qpel16_mc01_neon, export=1
443         vmov.i8         d0,  #52
444         vmov.i8         d1,  #20
445 .L\type\()_rv40_qpel16_v:
446         sub             r1,  r1,  r2,  lsl #1
447         push            {r1, lr}
448         vpush           {d8-d15}
449         bl              \type\()_rv40_qpel8_v_lowpass_neon
450         sub             r1,  r1,  r2,  lsl #2
451         bl              \type\()_rv40_qpel8_v_lowpass_neon
452         ldr             r1,  [sp, #64]
453         sub             r0,  r0,  r2,  lsl #4
454         add             r0,  r0,  #8
455         add             r1,  r1,  #8
456         bl              \type\()_rv40_qpel8_v_lowpass_neon
457         sub             r1,  r1,  r2,  lsl #2
458         bl              \type\()_rv40_qpel8_v_lowpass_neon
459         vpop            {d8-d15}
460         pop             {r1, pc}
461 endfunc
462
463 function ff_\type\()_rv40_qpel16_mc11_neon, export=1
464         sub             r1,  r1,  r2,  lsl #1
465         sub             r1,  r1,  #2
466         push            {r1, lr}
467         vpush           {d8-d15}
468         sub             sp,  sp,  #44*8
469         add             r12, sp,  #7
470         bic             r12, r12, #7
471         mov             r3,  #20
472         vmov.i8         d0,  #52
473         vmov.i8         d1,  #20
474         bl              put_rv40_qpel8_h_lp_packed_s6_neon
475         ldr             r1,  [sp, #416]
476         add             r1,  r1,  #8
477         mov             r3,  #20
478         bl              put_rv40_qpel8_h_lp_packed_s6_neon
479 .L\type\()_rv40_qpel16_v_s6:
480         add             r1,  sp,  #7
481         bic             r1,  r1,  #7
482         bl              \type\()_rv40_qpel8_v_lp_packed_s6_neon
483         sub             r1,  r1,  #40
484         bl              \type\()_rv40_qpel8_v_lp_packed_s6_neon
485         sub             r0,  r0,  r2,  lsl #4
486         add             r0,  r0,  #8
487         bl              \type\()_rv40_qpel8_v_lp_packed_s6_neon
488         sub             r1,  r1,  #40
489         bl              \type\()_rv40_qpel8_v_lp_packed_s6_neon
490         add             sp,  sp,  #44*8
491         vpop            {d8-d15}
492         pop             {r1, pc}
493 endfunc
494
495 function ff_\type\()_rv40_qpel16_mc21_neon, export=1
496         sub             r1,  r1,  r2,  lsl #1
497         sub             r1,  r1,  #2
498         push            {r1, lr}
499         vpush           {d8-d15}
500         sub             sp,  sp,  #44*8
501         add             r12, sp,  #7
502         bic             r12, r12, #7
503         mov             r3,  #20
504         vmov.i8         d0,  #20
505         vmov.i8         d1,  #20
506         bl              put_rv40_qpel8_h_lp_packed_s5_neon
507         ldr             r1,  [sp, #416]
508         add             r1,  r1,  #8
509         mov             r3,  #20
510         bl              put_rv40_qpel8_h_lp_packed_s5_neon
511         vmov.i8         d0,  #52
512         b               .L\type\()_rv40_qpel16_v_s6
513 endfunc
514
515 function ff_\type\()_rv40_qpel16_mc31_neon, export=1
516         sub             r1,  r1,  r2,  lsl #1
517         sub             r1,  r1,  #2
518         push            {r1, lr}
519         vpush           {d8-d15}
520         sub             sp,  sp,  #44*8
521         add             r12, sp,  #7
522         bic             r12, r12, #7
523         mov             r3,  #20
524         vmov.i8         d0,  #20
525         vmov.i8         d1,  #52
526         bl              put_rv40_qpel8_h_lp_packed_s6_neon
527         ldr             r1,  [sp, #416]
528         add             r1,  r1,  #8
529         mov             r3,  #20
530         bl              put_rv40_qpel8_h_lp_packed_s6_neon
531         vswp            d0,  d1
532         b               .L\type\()_rv40_qpel16_v_s6
533 endfunc
534
535 function ff_\type\()_rv40_qpel16_mc12_neon, export=1
536         sub             r1,  r1,  r2,  lsl #1
537         sub             r1,  r1,  #2
538         push            {r1, lr}
539         vpush           {d8-d15}
540         sub             sp,  sp,  #44*8
541         add             r12, sp,  #7
542         bic             r12, r12, #7
543         mov             r3,  #20
544         vmov.i8         d0,  #52
545         vmov.i8         d1,  #20
546         bl              put_rv40_qpel8_h_lp_packed_s6_neon
547         ldr             r1,  [sp, #416]
548         add             r1,  r1,  #8
549         mov             r3,  #20
550         bl              put_rv40_qpel8_h_lp_packed_s6_neon
551         vmov.i8         d0,  #20
552 .L\type\()_rv40_qpel16_v_s5:
553         add             r1,  sp,  #7
554         bic             r1,  r1,  #7
555         bl              \type\()_rv40_qpel8_v_lp_packed_s5_neon
556         sub             r1,  r1,  #40
557         bl              \type\()_rv40_qpel8_v_lp_packed_s5_neon
558         sub             r0,  r0,  r2,  lsl #4
559         add             r0,  r0,  #8
560         bl              \type\()_rv40_qpel8_v_lp_packed_s5_neon
561         sub             r1,  r1,  #40
562         bl              \type\()_rv40_qpel8_v_lp_packed_s5_neon
563         add             sp,  sp,  #44*8
564         vpop            {d8-d15}
565         pop             {r1, pc}
566 endfunc
567
568 function ff_\type\()_rv40_qpel16_mc22_neon, export=1
569         sub             r1,  r1,  r2,  lsl #1
570         sub             r1,  r1,  #2
571         push            {r1, lr}
572         vpush           {d8-d15}
573         sub             sp,  sp,  #44*8
574         add             r12, sp,  #7
575         bic             r12, r12, #7
576         mov             r3,  #20
577         vmov.i8         d0,  #20
578         vmov.i8         d1,  #20
579         bl              put_rv40_qpel8_h_lp_packed_s5_neon
580         ldr             r1,  [sp, #416]
581         add             r1,  r1,  #8
582         mov             r3,  #20
583         bl              put_rv40_qpel8_h_lp_packed_s5_neon
584         b               .L\type\()_rv40_qpel16_v_s5
585 endfunc
586
587 function ff_\type\()_rv40_qpel16_mc32_neon, export=1
588         sub             r1,  r1,  r2,  lsl #1
589         sub             r1,  r1,  #2
590         push            {r1, lr}
591         vpush           {d8-d15}
592         sub             sp,  sp,  #44*8
593         add             r12, sp,  #7
594         bic             r12, r12, #7
595         mov             r3,  #20
596         vmov.i8         d0,  #20
597         vmov.i8         d1,  #52
598         bl              put_rv40_qpel8_h_lp_packed_s6_neon
599         ldr             r1,  [sp, #416]
600         add             r1,  r1,  #8
601         mov             r3,  #20
602         bl              put_rv40_qpel8_h_lp_packed_s6_neon
603         vmov.i8         d1,  #20
604         b               .L\type\()_rv40_qpel16_v_s5
605 endfunc
606
607 function ff_\type\()_rv40_qpel16_mc03_neon, export=1
608         vmov.i8         d0,  #20
609         vmov.i8         d1,  #52
610         b               .L\type\()_rv40_qpel16_v
611 endfunc
612
613 function ff_\type\()_rv40_qpel16_mc13_neon, export=1
614         sub             r1,  r1,  r2,  lsl #1
615         sub             r1,  r1,  #2
616         push            {r1, lr}
617         vpush           {d8-d15}
618         sub             sp,  sp,  #44*8
619         add             r12, sp,  #7
620         bic             r12, r12, #7
621         mov             r3,  #20
622         vmov.i8         d0,  #52
623         vmov.i8         d1,  #20
624         bl              put_rv40_qpel8_h_lp_packed_s6_neon
625         ldr             r1,  [sp, #416]
626         add             r1,  r1,  #8
627         mov             r3,  #20
628         bl              put_rv40_qpel8_h_lp_packed_s6_neon
629         vswp            d0,  d1
630         b               .L\type\()_rv40_qpel16_v_s6
631 endfunc
632
633 function ff_\type\()_rv40_qpel16_mc23_neon, export=1
634         sub             r1,  r1,  r2,  lsl #1
635         sub             r1,  r1,  #2
636         push            {r1, lr}
637         vpush           {d8-d15}
638         sub             sp,  sp,  #44*8
639         add             r12, sp,  #7
640         bic             r12, r12, #7
641         mov             r3,  #20
642         vmov.i8         d0,  #20
643         vmov.i8         d1,  #20
644         bl              put_rv40_qpel8_h_lp_packed_s5_neon
645         ldr             r1,  [sp, #416]
646         add             r1,  r1,  #8
647         mov             r3,  #20
648         bl              put_rv40_qpel8_h_lp_packed_s5_neon
649         vmov.i8         d1,  #52
650         b               .L\type\()_rv40_qpel16_v_s6
651 endfunc
652
653 function ff_\type\()_rv40_qpel16_mc33_neon, export=1
654         mov             r3,  #16
655         b               ff_\type\()_pixels16_xy2_neon
656 endfunc
657 .endm
658
659         rv40_qpel       put
660         rv40_qpel       avg
661
662 .macro  rv40_weight
663         vmovl.u8        q8,  d2
664         vmovl.u8        q9,  d3
665         vmovl.u8        q10, d4
666         vmovl.u8        q11, d5
667         vmull.u16       q2,  d16, d0[2]
668         vmull.u16       q3,  d17, d0[2]
669         vmull.u16       q8,  d18, d0[2]
670         vmull.u16       q9,  d19, d0[2]
671         vmull.u16       q12, d20, d0[0]
672         vmull.u16       q13, d21, d0[0]
673         vmull.u16       q14, d22, d0[0]
674         vmull.u16       q15, d23, d0[0]
675         vshrn.i32       d4,  q2,  #9
676         vshrn.i32       d5,  q3,  #9
677         vshrn.i32       d6,  q8,  #9
678         vshrn.i32       d7,  q9,  #9
679         vshrn.i32       d16, q12, #9
680         vshrn.i32       d17, q13, #9
681         vshrn.i32       d18, q14, #9
682         vshrn.i32       d19, q15, #9
683         vadd.u16        q2,  q2,  q8
684         vadd.u16        q3,  q3,  q9
685         vrshrn.i16      d2,  q2,  #5
686         vrshrn.i16      d3,  q3,  #5
687 .endm
688
689 /* void ff_rv40_weight_func_16_neon(uint8_t *dst, uint8_t *src1, uint8_t *src2,
690                                     int w1, int w2, int stride) */
691 function ff_rv40_weight_func_16_neon, export=1
692         ldr             r12, [sp]
693         vmov            d0,  r3,  r12
694         ldr             r12, [sp, #4]
695         mov             r3,  #16
696 1:
697         vld1.8          {q1},     [r1,:128], r12
698         vld1.8          {q2},     [r2,:128], r12
699         rv40_weight
700         vst1.8          {q1},     [r0,:128], r12
701         subs            r3,  r3,  #1
702         bne             1b
703         bx              lr
704 endfunc
705
706 /* void ff_rv40_weight_func_8_neon(uint8_t *dst, uint8_t *src1, uint8_t *src2,
707                                    int w1, int w2, int stride) */
708 function ff_rv40_weight_func_8_neon, export=1
709         ldr             r12, [sp]
710         vmov            d0,  r3,  r12
711         ldr             r12, [sp, #4]
712         mov             r3,  #8
713 1:
714         vld1.8          {d2},     [r1,:64], r12
715         vld1.8          {d3},     [r1,:64], r12
716         vld1.8          {d4},     [r2,:64], r12
717         vld1.8          {d5},     [r2,:64], r12
718         rv40_weight
719         vst1.8          {d2},     [r0,:64], r12
720         vst1.8          {d3},     [r0,:64], r12
721         subs            r3,  r3,  #2
722         bne             1b
723         bx              lr
724 endfunc