Merge remote-tracking branch 'qatar/master'
[ffmpeg.git] / libavcodec / arm / dsputil_neon.S
1 /*
2  * ARM NEON optimised DSP functions
3  * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 #include "config.h"
23 #include "asm.S"
24
25         preserve8
26         .text
27
28 function ff_clear_block_neon, export=1
29         vmov.i16        q0,  #0
30         .rept           8
31         vst1.16         {q0}, [r0,:128]!
32         .endr
33         bx              lr
34 endfunc
35
36 function ff_clear_blocks_neon, export=1
37         vmov.i16        q0,  #0
38         .rept           8*6
39         vst1.16         {q0}, [r0,:128]!
40         .endr
41         bx              lr
42 endfunc
43
44         .macro pixels16 avg=0
45 .if \avg
46         mov             ip,  r0
47 .endif
48 1:      vld1.64         {d0, d1},  [r1], r2
49         vld1.64         {d2, d3},  [r1], r2
50         vld1.64         {d4, d5},  [r1], r2
51         pld             [r1, r2, lsl #2]
52         vld1.64         {d6, d7},  [r1], r2
53         pld             [r1]
54         pld             [r1, r2]
55         pld             [r1, r2, lsl #1]
56 .if \avg
57         vld1.64         {d16,d17}, [ip,:128], r2
58         vrhadd.u8       q0,  q0,  q8
59         vld1.64         {d18,d19}, [ip,:128], r2
60         vrhadd.u8       q1,  q1,  q9
61         vld1.64         {d20,d21}, [ip,:128], r2
62         vrhadd.u8       q2,  q2,  q10
63         vld1.64         {d22,d23}, [ip,:128], r2
64         vrhadd.u8       q3,  q3,  q11
65 .endif
66         subs            r3,  r3,  #4
67         vst1.64         {d0, d1},  [r0,:128], r2
68         vst1.64         {d2, d3},  [r0,:128], r2
69         vst1.64         {d4, d5},  [r0,:128], r2
70         vst1.64         {d6, d7},  [r0,:128], r2
71         bne             1b
72         bx              lr
73         .endm
74
75         .macro pixels16_x2 vhadd=vrhadd.u8
76 1:      vld1.64         {d0-d2},   [r1], r2
77         vld1.64         {d4-d6},   [r1], r2
78         pld             [r1]
79         pld             [r1, r2]
80         subs            r3,  r3,  #2
81         vext.8          q1,  q0,  q1,  #1
82         \vhadd          q0,  q0,  q1
83         vext.8          q3,  q2,  q3,  #1
84         \vhadd          q2,  q2,  q3
85         vst1.64         {d0, d1},  [r0,:128], r2
86         vst1.64         {d4, d5},  [r0,:128], r2
87         bne             1b
88         bx              lr
89         .endm
90
91         .macro pixels16_y2 vhadd=vrhadd.u8
92         vld1.64         {d0, d1},  [r1], r2
93         vld1.64         {d2, d3},  [r1], r2
94 1:      subs            r3,  r3,  #2
95         \vhadd          q2,  q0,  q1
96         vld1.64         {d0, d1},  [r1], r2
97         \vhadd          q3,  q0,  q1
98         vld1.64         {d2, d3},  [r1], r2
99         pld             [r1]
100         pld             [r1, r2]
101         vst1.64         {d4, d5},  [r0,:128], r2
102         vst1.64         {d6, d7},  [r0,:128], r2
103         bne             1b
104         bx              lr
105         .endm
106
107         .macro pixels16_xy2 vshrn=vrshrn.u16 no_rnd=0
108         vld1.64         {d0-d2},   [r1], r2
109         vld1.64         {d4-d6},   [r1], r2
110 .if \no_rnd
111         vmov.i16        q13, #1
112 .endif
113         pld             [r1]
114         pld             [r1, r2]
115         vext.8          q1,  q0,  q1,  #1
116         vext.8          q3,  q2,  q3,  #1
117         vaddl.u8        q8,  d0,  d2
118         vaddl.u8        q10, d1,  d3
119         vaddl.u8        q9,  d4,  d6
120         vaddl.u8        q11, d5,  d7
121 1:      subs            r3,  r3,  #2
122         vld1.64         {d0-d2},   [r1], r2
123         vadd.u16        q12, q8,  q9
124         pld             [r1]
125 .if \no_rnd
126         vadd.u16        q12, q12, q13
127 .endif
128         vext.8          q15, q0,  q1,  #1
129         vadd.u16        q1 , q10, q11
130         \vshrn          d28, q12, #2
131 .if \no_rnd
132         vadd.u16        q1,  q1,  q13
133 .endif
134         \vshrn          d29, q1,  #2
135         vaddl.u8        q8,  d0,  d30
136         vld1.64         {d2-d4},   [r1], r2
137         vaddl.u8        q10, d1,  d31
138         vst1.64         {d28,d29}, [r0,:128], r2
139         vadd.u16        q12, q8,  q9
140         pld             [r1, r2]
141 .if \no_rnd
142         vadd.u16        q12, q12, q13
143 .endif
144         vext.8          q2,  q1,  q2,  #1
145         vadd.u16        q0,  q10, q11
146         \vshrn          d30, q12, #2
147 .if \no_rnd
148         vadd.u16        q0,  q0,  q13
149 .endif
150         \vshrn          d31, q0,  #2
151         vaddl.u8        q9,  d2,  d4
152         vaddl.u8        q11, d3,  d5
153         vst1.64         {d30,d31}, [r0,:128], r2
154         bgt             1b
155         bx              lr
156         .endm
157
158         .macro pixels8 avg=0
159 1:      vld1.64         {d0}, [r1], r2
160         vld1.64         {d1}, [r1], r2
161         vld1.64         {d2}, [r1], r2
162         pld             [r1, r2, lsl #2]
163         vld1.64         {d3}, [r1], r2
164         pld             [r1]
165         pld             [r1, r2]
166         pld             [r1, r2, lsl #1]
167 .if \avg
168         vld1.64         {d4}, [r0,:64], r2
169         vrhadd.u8       d0,  d0,  d4
170         vld1.64         {d5}, [r0,:64], r2
171         vrhadd.u8       d1,  d1,  d5
172         vld1.64         {d6}, [r0,:64], r2
173         vrhadd.u8       d2,  d2,  d6
174         vld1.64         {d7}, [r0,:64], r2
175         vrhadd.u8       d3,  d3,  d7
176         sub             r0,  r0,  r2,  lsl #2
177 .endif
178         subs            r3,  r3,  #4
179         vst1.64         {d0}, [r0,:64], r2
180         vst1.64         {d1}, [r0,:64], r2
181         vst1.64         {d2}, [r0,:64], r2
182         vst1.64         {d3}, [r0,:64], r2
183         bne             1b
184         bx              lr
185         .endm
186
187         .macro pixels8_x2 vhadd=vrhadd.u8
188 1:      vld1.64         {d0, d1},  [r1], r2
189         vext.8          d1,  d0,  d1,  #1
190         vld1.64         {d2, d3},  [r1], r2
191         vext.8          d3,  d2,  d3,  #1
192         pld             [r1]
193         pld             [r1, r2]
194         subs            r3,  r3,  #2
195         vswp            d1,  d2
196         \vhadd          q0,  q0,  q1
197         vst1.64         {d0},      [r0,:64], r2
198         vst1.64         {d1},      [r0,:64], r2
199         bne             1b
200         bx              lr
201         .endm
202
203         .macro pixels8_y2 vhadd=vrhadd.u8
204         vld1.64         {d0},      [r1], r2
205         vld1.64         {d1},      [r1], r2
206 1:      subs            r3,  r3,  #2
207         \vhadd          d4,  d0,  d1
208         vld1.64         {d0},      [r1], r2
209         \vhadd          d5,  d0,  d1
210         vld1.64         {d1},      [r1], r2
211         pld             [r1]
212         pld             [r1, r2]
213         vst1.64         {d4},      [r0,:64], r2
214         vst1.64         {d5},      [r0,:64], r2
215         bne             1b
216         bx              lr
217         .endm
218
219         .macro pixels8_xy2 vshrn=vrshrn.u16 no_rnd=0
220         vld1.64         {d0, d1},  [r1], r2
221         vld1.64         {d2, d3},  [r1], r2
222 .if \no_rnd
223         vmov.i16        q11, #1
224 .endif
225         pld             [r1]
226         pld             [r1, r2]
227         vext.8          d4,  d0,  d1,  #1
228         vext.8          d6,  d2,  d3,  #1
229         vaddl.u8        q8,  d0,  d4
230         vaddl.u8        q9,  d2,  d6
231 1:      subs            r3,  r3,  #2
232         vld1.64         {d0, d1},  [r1], r2
233         pld             [r1]
234         vadd.u16        q10, q8,  q9
235         vext.8          d4,  d0,  d1,  #1
236 .if \no_rnd
237         vadd.u16        q10, q10, q11
238 .endif
239         vaddl.u8        q8,  d0,  d4
240         \vshrn          d5,  q10, #2
241         vld1.64         {d2, d3},  [r1], r2
242         vadd.u16        q10, q8,  q9
243         pld             [r1, r2]
244 .if \no_rnd
245         vadd.u16        q10, q10, q11
246 .endif
247         vst1.64         {d5},      [r0,:64], r2
248         \vshrn          d7,  q10, #2
249         vext.8          d6,  d2,  d3,  #1
250         vaddl.u8        q9,  d2,  d6
251         vst1.64         {d7},      [r0,:64], r2
252         bgt             1b
253         bx              lr
254         .endm
255
256         .macro pixfunc pfx name suf rnd_op args:vararg
257 function ff_\pfx\name\suf\()_neon, export=1
258         \name \rnd_op \args
259 endfunc
260         .endm
261
262         .macro pixfunc2 pfx name args:vararg
263         pixfunc \pfx \name
264         pixfunc \pfx \name \args
265         .endm
266
267 function ff_put_h264_qpel16_mc00_neon, export=1
268         mov             r3,  #16
269 endfunc
270
271         pixfunc  put_ pixels16
272         pixfunc2 put_ pixels16_x2,  _no_rnd, vhadd.u8
273         pixfunc2 put_ pixels16_y2,  _no_rnd, vhadd.u8
274         pixfunc2 put_ pixels16_xy2, _no_rnd, vshrn.u16, 1
275
276 function ff_avg_h264_qpel16_mc00_neon, export=1
277         mov             r3,  #16
278 endfunc
279
280         pixfunc  avg_ pixels16,, 1
281
282 function ff_put_h264_qpel8_mc00_neon, export=1
283         mov             r3,  #8
284 endfunc
285
286         pixfunc  put_ pixels8
287         pixfunc2 put_ pixels8_x2,   _no_rnd, vhadd.u8
288         pixfunc2 put_ pixels8_y2,   _no_rnd, vhadd.u8
289         pixfunc2 put_ pixels8_xy2,  _no_rnd, vshrn.u16, 1
290
291 function ff_avg_h264_qpel8_mc00_neon, export=1
292         mov             r3,  #8
293 endfunc
294
295         pixfunc  avg_ pixels8,, 1
296
297 function ff_put_pixels_clamped_neon, export=1
298         vld1.64         {d16-d19}, [r0,:128]!
299         vqmovun.s16     d0, q8
300         vld1.64         {d20-d23}, [r0,:128]!
301         vqmovun.s16     d1, q9
302         vld1.64         {d24-d27}, [r0,:128]!
303         vqmovun.s16     d2, q10
304         vld1.64         {d28-d31}, [r0,:128]!
305         vqmovun.s16     d3, q11
306         vst1.64         {d0},      [r1,:64], r2
307         vqmovun.s16     d4, q12
308         vst1.64         {d1},      [r1,:64], r2
309         vqmovun.s16     d5, q13
310         vst1.64         {d2},      [r1,:64], r2
311         vqmovun.s16     d6, q14
312         vst1.64         {d3},      [r1,:64], r2
313         vqmovun.s16     d7, q15
314         vst1.64         {d4},      [r1,:64], r2
315         vst1.64         {d5},      [r1,:64], r2
316         vst1.64         {d6},      [r1,:64], r2
317         vst1.64         {d7},      [r1,:64], r2
318         bx              lr
319 endfunc
320
321 function ff_put_signed_pixels_clamped_neon, export=1
322         vmov.u8         d31, #128
323         vld1.64         {d16-d17}, [r0,:128]!
324         vqmovn.s16      d0, q8
325         vld1.64         {d18-d19}, [r0,:128]!
326         vqmovn.s16      d1, q9
327         vld1.64         {d16-d17}, [r0,:128]!
328         vqmovn.s16      d2, q8
329         vld1.64         {d18-d19}, [r0,:128]!
330         vadd.u8         d0, d0, d31
331         vld1.64         {d20-d21}, [r0,:128]!
332         vadd.u8         d1, d1, d31
333         vld1.64         {d22-d23}, [r0,:128]!
334         vadd.u8         d2, d2, d31
335         vst1.64         {d0},      [r1,:64], r2
336         vqmovn.s16      d3, q9
337         vst1.64         {d1},      [r1,:64], r2
338         vqmovn.s16      d4, q10
339         vst1.64         {d2},      [r1,:64], r2
340         vqmovn.s16      d5, q11
341         vld1.64         {d24-d25}, [r0,:128]!
342         vadd.u8         d3, d3, d31
343         vld1.64         {d26-d27}, [r0,:128]!
344         vadd.u8         d4, d4, d31
345         vadd.u8         d5, d5, d31
346         vst1.64         {d3},      [r1,:64], r2
347         vqmovn.s16      d6, q12
348         vst1.64         {d4},      [r1,:64], r2
349         vqmovn.s16      d7, q13
350         vst1.64         {d5},      [r1,:64], r2
351         vadd.u8         d6, d6, d31
352         vadd.u8         d7, d7, d31
353         vst1.64         {d6},      [r1,:64], r2
354         vst1.64         {d7},      [r1,:64], r2
355         bx              lr
356 endfunc
357
358 function ff_add_pixels_clamped_neon, export=1
359         mov             r3, r1
360         vld1.64         {d16},   [r1,:64], r2
361         vld1.64         {d0-d1}, [r0,:128]!
362         vaddw.u8        q0, q0, d16
363         vld1.64         {d17},   [r1,:64], r2
364         vld1.64         {d2-d3}, [r0,:128]!
365         vqmovun.s16     d0, q0
366         vld1.64         {d18},   [r1,:64], r2
367         vaddw.u8        q1, q1, d17
368         vld1.64         {d4-d5}, [r0,:128]!
369         vaddw.u8        q2, q2, d18
370         vst1.64         {d0},    [r3,:64], r2
371         vqmovun.s16     d2, q1
372         vld1.64         {d19},   [r1,:64], r2
373         vld1.64         {d6-d7}, [r0,:128]!
374         vaddw.u8        q3, q3, d19
375         vqmovun.s16     d4, q2
376         vst1.64         {d2},    [r3,:64], r2
377         vld1.64         {d16},   [r1,:64], r2
378         vqmovun.s16     d6, q3
379         vld1.64         {d0-d1}, [r0,:128]!
380         vaddw.u8        q0, q0, d16
381         vst1.64         {d4},    [r3,:64], r2
382         vld1.64         {d17},   [r1,:64], r2
383         vld1.64         {d2-d3}, [r0,:128]!
384         vaddw.u8        q1, q1, d17
385         vst1.64         {d6},    [r3,:64], r2
386         vqmovun.s16     d0, q0
387         vld1.64         {d18},   [r1,:64], r2
388         vld1.64         {d4-d5}, [r0,:128]!
389         vaddw.u8        q2, q2, d18
390         vst1.64         {d0},    [r3,:64], r2
391         vqmovun.s16     d2, q1
392         vld1.64         {d19},   [r1,:64], r2
393         vqmovun.s16     d4, q2
394         vld1.64         {d6-d7}, [r0,:128]!
395         vaddw.u8        q3, q3, d19
396         vst1.64         {d2},    [r3,:64], r2
397         vqmovun.s16     d6, q3
398         vst1.64         {d4},    [r3,:64], r2
399         vst1.64         {d6},    [r3,:64], r2
400         bx              lr
401 endfunc
402
403 function ff_vector_fmul_neon, export=1
404         subs            r3,  r3,  #8
405         vld1.64         {d0-d3},  [r1,:128]!
406         vld1.64         {d4-d7},  [r2,:128]!
407         vmul.f32        q8,  q0,  q2
408         vmul.f32        q9,  q1,  q3
409         beq             3f
410         bics            ip,  r3,  #15
411         beq             2f
412 1:      subs            ip,  ip,  #16
413         vld1.64         {d0-d1},  [r1,:128]!
414         vld1.64         {d4-d5},  [r2,:128]!
415         vmul.f32        q10, q0,  q2
416         vld1.64         {d2-d3},  [r1,:128]!
417         vld1.64         {d6-d7},  [r2,:128]!
418         vmul.f32        q11, q1,  q3
419         vst1.64         {d16-d19},[r0,:128]!
420         vld1.64         {d0-d1},  [r1,:128]!
421         vld1.64         {d4-d5},  [r2,:128]!
422         vmul.f32        q8,  q0,  q2
423         vld1.64         {d2-d3},  [r1,:128]!
424         vld1.64         {d6-d7},  [r2,:128]!
425         vmul.f32        q9,  q1,  q3
426         vst1.64         {d20-d23},[r0,:128]!
427         bne             1b
428         ands            r3,  r3,  #15
429         beq             3f
430 2:      vld1.64         {d0-d1},  [r1,:128]!
431         vld1.64         {d4-d5},  [r2,:128]!
432         vst1.64         {d16-d17},[r0,:128]!
433         vmul.f32        q8,  q0,  q2
434         vld1.64         {d2-d3},  [r1,:128]!
435         vld1.64         {d6-d7},  [r2,:128]!
436         vst1.64         {d18-d19},[r0,:128]!
437         vmul.f32        q9,  q1,  q3
438 3:      vst1.64         {d16-d19},[r0,:128]!
439         bx              lr
440 endfunc
441
442 function ff_vector_fmul_window_neon, export=1
443         push            {r4,r5,lr}
444         ldr             lr,  [sp, #12]
445         sub             r2,  r2,  #8
446         sub             r5,  lr,  #2
447         add             r2,  r2,  r5, lsl #2
448         add             r4,  r3,  r5, lsl #3
449         add             ip,  r0,  r5, lsl #3
450         mov             r5,  #-16
451         vld1.64         {d0,d1},  [r1,:128]!
452         vld1.64         {d2,d3},  [r2,:128], r5
453         vld1.64         {d4,d5},  [r3,:128]!
454         vld1.64         {d6,d7},  [r4,:128], r5
455 1:      subs            lr,  lr,  #4
456         vmul.f32        d22, d0,  d4
457         vrev64.32       q3,  q3
458         vmul.f32        d23, d1,  d5
459         vrev64.32       q1,  q1
460         vmul.f32        d20, d0,  d7
461         vmul.f32        d21, d1,  d6
462         beq             2f
463         vmla.f32        d22, d3,  d7
464         vld1.64         {d0,d1},  [r1,:128]!
465         vmla.f32        d23, d2,  d6
466         vld1.64         {d18,d19},[r2,:128], r5
467         vmls.f32        d20, d3,  d4
468         vld1.64         {d24,d25},[r3,:128]!
469         vmls.f32        d21, d2,  d5
470         vld1.64         {d6,d7},  [r4,:128], r5
471         vmov            q1,  q9
472         vrev64.32       q11, q11
473         vmov            q2,  q12
474         vswp            d22, d23
475         vst1.64         {d20,d21},[r0,:128]!
476         vst1.64         {d22,d23},[ip,:128], r5
477         b               1b
478 2:      vmla.f32        d22, d3,  d7
479         vmla.f32        d23, d2,  d6
480         vmls.f32        d20, d3,  d4
481         vmls.f32        d21, d2,  d5
482         vrev64.32       q11, q11
483         vswp            d22, d23
484         vst1.64         {d20,d21},[r0,:128]!
485         vst1.64         {d22,d23},[ip,:128], r5
486         pop             {r4,r5,pc}
487 endfunc
488
489 #if CONFIG_VORBIS_DECODER
490 function ff_vorbis_inverse_coupling_neon, export=1
491         vmov.i32        q10, #1<<31
492         subs            r2,  r2,  #4
493         mov             r3,  r0
494         mov             r12, r1
495         beq             3f
496
497         vld1.32         {d24-d25},[r1,:128]!
498         vld1.32         {d22-d23},[r0,:128]!
499         vcle.s32        q8,  q12, #0
500         vand            q9,  q11, q10
501         veor            q12, q12, q9
502         vand            q2,  q12, q8
503         vbic            q3,  q12, q8
504         vadd.f32        q12, q11, q2
505         vsub.f32        q11, q11, q3
506 1:      vld1.32         {d2-d3},  [r1,:128]!
507         vld1.32         {d0-d1},  [r0,:128]!
508         vcle.s32        q8,  q1,  #0
509         vand            q9,  q0,  q10
510         veor            q1,  q1,  q9
511         vst1.32         {d24-d25},[r3, :128]!
512         vst1.32         {d22-d23},[r12,:128]!
513         vand            q2,  q1,  q8
514         vbic            q3,  q1,  q8
515         vadd.f32        q1,  q0,  q2
516         vsub.f32        q0,  q0,  q3
517         subs            r2,  r2,  #8
518         ble             2f
519         vld1.32         {d24-d25},[r1,:128]!
520         vld1.32         {d22-d23},[r0,:128]!
521         vcle.s32        q8,  q12, #0
522         vand            q9,  q11, q10
523         veor            q12, q12, q9
524         vst1.32         {d2-d3},  [r3, :128]!
525         vst1.32         {d0-d1},  [r12,:128]!
526         vand            q2,  q12, q8
527         vbic            q3,  q12, q8
528         vadd.f32        q12, q11, q2
529         vsub.f32        q11, q11, q3
530         b               1b
531
532 2:      vst1.32         {d2-d3},  [r3, :128]!
533         vst1.32         {d0-d1},  [r12,:128]!
534         it              lt
535         bxlt            lr
536
537 3:      vld1.32         {d2-d3},  [r1,:128]
538         vld1.32         {d0-d1},  [r0,:128]
539         vcle.s32        q8,  q1,  #0
540         vand            q9,  q0,  q10
541         veor            q1,  q1,  q9
542         vand            q2,  q1,  q8
543         vbic            q3,  q1,  q8
544         vadd.f32        q1,  q0,  q2
545         vsub.f32        q0,  q0,  q3
546         vst1.32         {d2-d3},  [r0,:128]!
547         vst1.32         {d0-d1},  [r1,:128]!
548         bx              lr
549 endfunc
550 #endif
551
552 function ff_vector_fmul_scalar_neon, export=1
553 VFP     len .req r2
554 NOVFP   len .req r3
555 VFP     vdup.32         q8,  d0[0]
556 NOVFP   vdup.32         q8,  r2
557         bics            r12, len, #15
558         beq             3f
559         vld1.32         {q0},[r1,:128]!
560         vld1.32         {q1},[r1,:128]!
561 1:      vmul.f32        q0,  q0,  q8
562         vld1.32         {q2},[r1,:128]!
563         vmul.f32        q1,  q1,  q8
564         vld1.32         {q3},[r1,:128]!
565         vmul.f32        q2,  q2,  q8
566         vst1.32         {q0},[r0,:128]!
567         vmul.f32        q3,  q3,  q8
568         vst1.32         {q1},[r0,:128]!
569         subs            r12, r12, #16
570         beq             2f
571         vld1.32         {q0},[r1,:128]!
572         vst1.32         {q2},[r0,:128]!
573         vld1.32         {q1},[r1,:128]!
574         vst1.32         {q3},[r0,:128]!
575         b               1b
576 2:      vst1.32         {q2},[r0,:128]!
577         vst1.32         {q3},[r0,:128]!
578         ands            len, len, #15
579         it              eq
580         bxeq            lr
581 3:      vld1.32         {q0},[r1,:128]!
582         vmul.f32        q0,  q0,  q8
583         vst1.32         {q0},[r0,:128]!
584         subs            len, len, #4
585         bgt             3b
586         bx              lr
587         .unreq          len
588 endfunc
589
590 function ff_vector_fmul_sv_scalar_2_neon, export=1
591 VFP     vdup.32         d16, d0[0]
592 NOVFP   vdup.32         d16, r3
593 NOVFP   ldr             r3,  [sp]
594         vld1.32         {d0},[r1,:64]!
595         vld1.32         {d1},[r1,:64]!
596 1:      subs            r3,  r3,  #4
597         vmul.f32        d4,  d0,  d16
598         vmul.f32        d5,  d1,  d16
599         ldr             r12, [r2], #4
600         vld1.32         {d2},[r12,:64]
601         ldr             r12, [r2], #4
602         vld1.32         {d3},[r12,:64]
603         vmul.f32        d4,  d4,  d2
604         vmul.f32        d5,  d5,  d3
605         beq             2f
606         vld1.32         {d0},[r1,:64]!
607         vld1.32         {d1},[r1,:64]!
608         vst1.32         {d4},[r0,:64]!
609         vst1.32         {d5},[r0,:64]!
610         b               1b
611 2:      vst1.32         {d4},[r0,:64]!
612         vst1.32         {d5},[r0,:64]!
613         bx              lr
614 endfunc
615
616 function ff_vector_fmul_sv_scalar_4_neon, export=1
617 VFP     vdup.32         q10, d0[0]
618 NOVFP   vdup.32         q10, r3
619 NOVFP   ldr             r3,  [sp]
620         push            {lr}
621         bics            lr,  r3,  #7
622         beq             3f
623         vld1.32         {q0},[r1,:128]!
624         vld1.32         {q2},[r1,:128]!
625 1:      ldr             r12, [r2], #4
626         vld1.32         {q1},[r12,:128]
627         ldr             r12, [r2], #4
628         vld1.32         {q3},[r12,:128]
629         vmul.f32        q8,  q0,  q10
630         vmul.f32        q8,  q8,  q1
631         vmul.f32        q9,  q2,  q10
632         vmul.f32        q9,  q9,  q3
633         subs            lr,  lr,  #8
634         beq             2f
635         vld1.32         {q0},[r1,:128]!
636         vld1.32         {q2},[r1,:128]!
637         vst1.32         {q8},[r0,:128]!
638         vst1.32         {q9},[r0,:128]!
639         b               1b
640 2:      vst1.32         {q8},[r0,:128]!
641         vst1.32         {q9},[r0,:128]!
642         ands            r3,  r3,  #7
643         it              eq
644         popeq           {pc}
645 3:      vld1.32         {q0},[r1,:128]!
646         ldr             r12, [r2], #4
647         vld1.32         {q1},[r12,:128]
648         vmul.f32        q0,  q0,  q10
649         vmul.f32        q0,  q0,  q1
650         vst1.32         {q0},[r0,:128]!
651         subs            r3,  r3,  #4
652         bgt             3b
653         pop             {pc}
654 endfunc
655
656 function ff_sv_fmul_scalar_2_neon, export=1
657 VFP     len .req r2
658 NOVFP   len .req r3
659 VFP     vdup.32         q8,  d0[0]
660 NOVFP   vdup.32         q8,  r2
661         ldr             r12, [r1], #4
662         vld1.32         {d0},[r12,:64]
663         ldr             r12, [r1], #4
664         vld1.32         {d1},[r12,:64]
665 1:      vmul.f32        q1,  q0,  q8
666         subs            len, len, #4
667         beq             2f
668         ldr             r12, [r1], #4
669         vld1.32         {d0},[r12,:64]
670         ldr             r12, [r1], #4
671         vld1.32         {d1},[r12,:64]
672         vst1.32         {q1},[r0,:128]!
673         b               1b
674 2:      vst1.32         {q1},[r0,:128]!
675         bx              lr
676         .unreq          len
677 endfunc
678
679 function ff_sv_fmul_scalar_4_neon, export=1
680 VFP     len .req r2
681 NOVFP   len .req r3
682 VFP     vdup.32         q8,  d0[0]
683 NOVFP   vdup.32         q8,  r2
684 1:      ldr             r12, [r1], #4
685         vld1.32         {q0},[r12,:128]
686         vmul.f32        q0,  q0,  q8
687         vst1.32         {q0},[r0,:128]!
688         subs            len, len, #4
689         bgt             1b
690         bx              lr
691         .unreq          len
692 endfunc
693
694 function ff_butterflies_float_neon, export=1
695 1:      vld1.32         {q0},[r0,:128]
696         vld1.32         {q1},[r1,:128]
697         vsub.f32        q2,  q0,  q1
698         vadd.f32        q1,  q0,  q1
699         vst1.32         {q2},[r1,:128]!
700         vst1.32         {q1},[r0,:128]!
701         subs            r2,  r2,  #4
702         bgt             1b
703         bx              lr
704 endfunc
705
706 function ff_scalarproduct_float_neon, export=1
707         vmov.f32        q2,  #0.0
708 1:      vld1.32         {q0},[r0,:128]!
709         vld1.32         {q1},[r1,:128]!
710         vmla.f32        q2,  q0,  q1
711         subs            r2,  r2,  #4
712         bgt             1b
713         vadd.f32        d0,  d4,  d5
714         vpadd.f32       d0,  d0,  d0
715 NOVFP   vmov.32         r0,  d0[0]
716         bx              lr
717 endfunc
718
719 function ff_vector_fmul_reverse_neon, export=1
720         add             r2,  r2,  r3,  lsl #2
721         sub             r2,  r2,  #32
722         mov             r12, #-32
723         vld1.32         {q0-q1},  [r1,:128]!
724         vld1.32         {q2-q3},  [r2,:128], r12
725 1:      pld             [r1, #32]
726         vrev64.32       q3,  q3
727         vmul.f32        d16, d0,  d7
728         vmul.f32        d17, d1,  d6
729         pld             [r2, #-32]
730         vrev64.32       q2,  q2
731         vmul.f32        d18, d2,  d5
732         vmul.f32        d19, d3,  d4
733         subs            r3,  r3,  #8
734         beq             2f
735         vld1.32         {q0-q1},  [r1,:128]!
736         vld1.32         {q2-q3},  [r2,:128], r12
737         vst1.32         {q8-q9},  [r0,:128]!
738         b               1b
739 2:      vst1.32         {q8-q9},  [r0,:128]!
740         bx              lr
741 endfunc
742
743 function ff_vector_fmul_add_neon, export=1
744         ldr             r12, [sp]
745         vld1.32         {q0-q1},  [r1,:128]!
746         vld1.32         {q8-q9},  [r2,:128]!
747         vld1.32         {q2-q3},  [r3,:128]!
748         vmul.f32        q10, q0,  q8
749         vmul.f32        q11, q1,  q9
750 1:      vadd.f32        q12, q2,  q10
751         vadd.f32        q13, q3,  q11
752         pld             [r1, #16]
753         pld             [r2, #16]
754         pld             [r3, #16]
755         subs            r12, r12, #8
756         beq             2f
757         vld1.32         {q0},     [r1,:128]!
758         vld1.32         {q8},     [r2,:128]!
759         vmul.f32        q10, q0,  q8
760         vld1.32         {q1},     [r1,:128]!
761         vld1.32         {q9},     [r2,:128]!
762         vmul.f32        q11, q1,  q9
763         vld1.32         {q2-q3},  [r3,:128]!
764         vst1.32         {q12-q13},[r0,:128]!
765         b               1b
766 2:      vst1.32         {q12-q13},[r0,:128]!
767         bx              lr
768 endfunc
769
770 function ff_vector_clipf_neon, export=1
771 VFP     vdup.32         q1,  d0[1]
772 VFP     vdup.32         q0,  d0[0]
773 NOVFP   vdup.32         q0,  r2
774 NOVFP   vdup.32         q1,  r3
775 NOVFP   ldr             r2,  [sp]
776         vld1.f32        {q2},[r1,:128]!
777         vmin.f32        q10, q2,  q1
778         vld1.f32        {q3},[r1,:128]!
779         vmin.f32        q11, q3,  q1
780 1:      vmax.f32        q8,  q10, q0
781         vmax.f32        q9,  q11, q0
782         subs            r2,  r2,  #8
783         beq             2f
784         vld1.f32        {q2},[r1,:128]!
785         vmin.f32        q10, q2,  q1
786         vld1.f32        {q3},[r1,:128]!
787         vmin.f32        q11, q3,  q1
788         vst1.f32        {q8},[r0,:128]!
789         vst1.f32        {q9},[r0,:128]!
790         b               1b
791 2:      vst1.f32        {q8},[r0,:128]!
792         vst1.f32        {q9},[r0,:128]!
793         bx              lr
794 endfunc
795
796 function ff_apply_window_int16_neon, export=1
797         push            {r4,lr}
798         add             r4,  r1,  r3,  lsl #1
799         add             lr,  r0,  r3,  lsl #1
800         sub             r4,  r4,  #16
801         sub             lr,  lr,  #16
802         mov             r12, #-16
803 1:
804         vld1.16         {q0},     [r1,:128]!
805         vld1.16         {q2},     [r2,:128]!
806         vld1.16         {q1},     [r4,:128], r12
807         vrev64.16       q3,  q2
808         vqrdmulh.s16    q0,  q0,  q2
809         vqrdmulh.s16    d2,  d2,  d7
810         vqrdmulh.s16    d3,  d3,  d6
811         vst1.16         {q0},     [r0,:128]!
812         vst1.16         {q1},     [lr,:128], r12
813         subs            r3,  r3,  #16
814         bgt             1b
815
816         pop             {r4,pc}
817 endfunc
818
819 function ff_vector_clip_int32_neon, export=1
820         vdup.32         q0,  r2
821         vdup.32         q1,  r3
822         ldr             r2,  [sp]
823 1:
824         vld1.32         {q2-q3},  [r1,:128]!
825         vmin.s32        q2,  q2,  q1
826         vmin.s32        q3,  q3,  q1
827         vmax.s32        q2,  q2,  q0
828         vmax.s32        q3,  q3,  q0
829         vst1.32         {q2-q3},  [r0,:128]!
830         subs            r2,  r2,  #8
831         bgt             1b
832         bx              lr
833 endfunc