ARM: whitespace cosmetics
[ffmpeg.git] / libavcodec / arm / dsputil_neon.S
1 /*
2  * ARM NEON optimised DSP functions
3  * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 #include "config.h"
23 #include "asm.S"
24
25         preserve8
26         .text
27
28         .macro pixels16 avg=0
29 .if \avg
30         mov             ip,  r0
31 .endif
32 1:      vld1.64         {d0, d1},  [r1], r2
33         vld1.64         {d2, d3},  [r1], r2
34         vld1.64         {d4, d5},  [r1], r2
35         pld             [r1, r2, lsl #2]
36         vld1.64         {d6, d7},  [r1], r2
37         pld             [r1]
38         pld             [r1, r2]
39         pld             [r1, r2, lsl #1]
40 .if \avg
41         vld1.64         {d16,d17}, [ip,:128], r2
42         vrhadd.u8       q0,  q0,  q8
43         vld1.64         {d18,d19}, [ip,:128], r2
44         vrhadd.u8       q1,  q1,  q9
45         vld1.64         {d20,d21}, [ip,:128], r2
46         vrhadd.u8       q2,  q2,  q10
47         vld1.64         {d22,d23}, [ip,:128], r2
48         vrhadd.u8       q3,  q3,  q11
49 .endif
50         subs            r3,  r3,  #4
51         vst1.64         {d0, d1},  [r0,:128], r2
52         vst1.64         {d2, d3},  [r0,:128], r2
53         vst1.64         {d4, d5},  [r0,:128], r2
54         vst1.64         {d6, d7},  [r0,:128], r2
55         bne             1b
56         bx              lr
57         .endm
58
59         .macro pixels16_x2 vhadd=vrhadd.u8
60 1:      vld1.64         {d0-d2},   [r1], r2
61         vld1.64         {d4-d6},   [r1], r2
62         pld             [r1]
63         pld             [r1, r2]
64         subs            r3,  r3,  #2
65         vext.8          q1,  q0,  q1,  #1
66         \vhadd          q0,  q0,  q1
67         vext.8          q3,  q2,  q3,  #1
68         \vhadd          q2,  q2,  q3
69         vst1.64         {d0, d1},  [r0,:128], r2
70         vst1.64         {d4, d5},  [r0,:128], r2
71         bne             1b
72         bx              lr
73         .endm
74
75         .macro pixels16_y2 vhadd=vrhadd.u8
76         vld1.64         {d0, d1},  [r1], r2
77         vld1.64         {d2, d3},  [r1], r2
78 1:      subs            r3,  r3,  #2
79         \vhadd          q2,  q0,  q1
80         vld1.64         {d0, d1},  [r1], r2
81         \vhadd          q3,  q0,  q1
82         vld1.64         {d2, d3},  [r1], r2
83         pld             [r1]
84         pld             [r1, r2]
85         vst1.64         {d4, d5},  [r0,:128], r2
86         vst1.64         {d6, d7},  [r0,:128], r2
87         bne             1b
88         bx              lr
89         .endm
90
91         .macro pixels16_xy2 vshrn=vrshrn.u16 no_rnd=0
92         vld1.64         {d0-d2},   [r1], r2
93         vld1.64         {d4-d6},   [r1], r2
94 .if \no_rnd
95         vmov.i16        q13, #1
96 .endif
97         pld             [r1]
98         pld             [r1, r2]
99         vext.8          q1,  q0,  q1,  #1
100         vext.8          q3,  q2,  q3,  #1
101         vaddl.u8        q8,  d0,  d2
102         vaddl.u8        q10, d1,  d3
103         vaddl.u8        q9,  d4,  d6
104         vaddl.u8        q11, d5,  d7
105 1:      subs            r3,  r3,  #2
106         vld1.64         {d0-d2},   [r1], r2
107         vadd.u16        q12, q8,  q9
108         pld             [r1]
109 .if \no_rnd
110         vadd.u16        q12, q12, q13
111 .endif
112         vext.8          q15, q0,  q1,  #1
113         vadd.u16        q1 , q10, q11
114         \vshrn          d28, q12, #2
115 .if \no_rnd
116         vadd.u16        q1,  q1,  q13
117 .endif
118         \vshrn          d29, q1,  #2
119         vaddl.u8        q8,  d0,  d30
120         vld1.64         {d2-d4},   [r1], r2
121         vaddl.u8        q10, d1,  d31
122         vst1.64         {d28,d29}, [r0,:128], r2
123         vadd.u16        q12, q8,  q9
124         pld             [r1, r2]
125 .if \no_rnd
126         vadd.u16        q12, q12, q13
127 .endif
128         vext.8          q2,  q1,  q2,  #1
129         vadd.u16        q0,  q10, q11
130         \vshrn          d30, q12, #2
131 .if \no_rnd
132         vadd.u16        q0,  q0,  q13
133 .endif
134         \vshrn          d31, q0,  #2
135         vaddl.u8        q9,  d2,  d4
136         vaddl.u8        q11, d3,  d5
137         vst1.64         {d30,d31}, [r0,:128], r2
138         bgt             1b
139         bx              lr
140         .endm
141
142         .macro pixels8 avg=0
143 1:      vld1.64         {d0}, [r1], r2
144         vld1.64         {d1}, [r1], r2
145         vld1.64         {d2}, [r1], r2
146         pld             [r1, r2, lsl #2]
147         vld1.64         {d3}, [r1], r2
148         pld             [r1]
149         pld             [r1, r2]
150         pld             [r1, r2, lsl #1]
151 .if \avg
152         vld1.64         {d4}, [r0,:64], r2
153         vrhadd.u8       d0,  d0,  d4
154         vld1.64         {d5}, [r0,:64], r2
155         vrhadd.u8       d1,  d1,  d5
156         vld1.64         {d6}, [r0,:64], r2
157         vrhadd.u8       d2,  d2,  d6
158         vld1.64         {d7}, [r0,:64], r2
159         vrhadd.u8       d3,  d3,  d7
160         sub             r0,  r0,  r2,  lsl #2
161 .endif
162         subs            r3,  r3,  #4
163         vst1.64         {d0}, [r0,:64], r2
164         vst1.64         {d1}, [r0,:64], r2
165         vst1.64         {d2}, [r0,:64], r2
166         vst1.64         {d3}, [r0,:64], r2
167         bne             1b
168         bx              lr
169         .endm
170
171         .macro pixels8_x2 vhadd=vrhadd.u8
172 1:      vld1.64         {d0, d1},  [r1], r2
173         vext.8          d1,  d0,  d1,  #1
174         vld1.64         {d2, d3},  [r1], r2
175         vext.8          d3,  d2,  d3,  #1
176         pld             [r1]
177         pld             [r1, r2]
178         subs            r3,  r3,  #2
179         vswp            d1,  d2
180         \vhadd          q0,  q0,  q1
181         vst1.64         {d0},      [r0,:64], r2
182         vst1.64         {d1},      [r0,:64], r2
183         bne             1b
184         bx              lr
185         .endm
186
187         .macro pixels8_y2 vhadd=vrhadd.u8
188         vld1.64         {d0},      [r1], r2
189         vld1.64         {d1},      [r1], r2
190 1:      subs            r3,  r3,  #2
191         \vhadd          d4,  d0,  d1
192         vld1.64         {d0},      [r1], r2
193         \vhadd          d5,  d0,  d1
194         vld1.64         {d1},      [r1], r2
195         pld             [r1]
196         pld             [r1, r2]
197         vst1.64         {d4},      [r0,:64], r2
198         vst1.64         {d5},      [r0,:64], r2
199         bne             1b
200         bx              lr
201         .endm
202
203         .macro pixels8_xy2 vshrn=vrshrn.u16 no_rnd=0
204         vld1.64         {d0, d1},  [r1], r2
205         vld1.64         {d2, d3},  [r1], r2
206 .if \no_rnd
207         vmov.i16        q11, #1
208 .endif
209         pld             [r1]
210         pld             [r1, r2]
211         vext.8          d4,  d0,  d1,  #1
212         vext.8          d6,  d2,  d3,  #1
213         vaddl.u8        q8,  d0,  d4
214         vaddl.u8        q9,  d2,  d6
215 1:      subs            r3,  r3,  #2
216         vld1.64         {d0, d1},  [r1], r2
217         pld             [r1]
218         vadd.u16        q10, q8,  q9
219         vext.8          d4,  d0,  d1,  #1
220 .if \no_rnd
221         vadd.u16        q10, q10, q11
222 .endif
223         vaddl.u8        q8,  d0,  d4
224         \vshrn          d5,  q10, #2
225         vld1.64         {d2, d3},  [r1], r2
226         vadd.u16        q10, q8,  q9
227         pld             [r1, r2]
228 .if \no_rnd
229         vadd.u16        q10, q10, q11
230 .endif
231         vst1.64         {d5},      [r0,:64], r2
232         \vshrn          d7,  q10, #2
233         vext.8          d6,  d2,  d3,  #1
234         vaddl.u8        q9,  d2,  d6
235         vst1.64         {d7},      [r0,:64], r2
236         bgt             1b
237         bx              lr
238         .endm
239
240         .macro pixfunc pfx name suf rnd_op args:vararg
241 function ff_\pfx\name\suf\()_neon, export=1
242         \name \rnd_op \args
243         .endfunc
244         .endm
245
246         .macro pixfunc2 pfx name args:vararg
247         pixfunc \pfx \name
248         pixfunc \pfx \name \args
249         .endm
250
251 function ff_put_h264_qpel16_mc00_neon, export=1
252         mov             r3,  #16
253         .endfunc
254
255         pixfunc  put_ pixels16
256         pixfunc2 put_ pixels16_x2,  _no_rnd, vhadd.u8
257         pixfunc2 put_ pixels16_y2,  _no_rnd, vhadd.u8
258         pixfunc2 put_ pixels16_xy2, _no_rnd, vshrn.u16, 1
259
260 function ff_avg_h264_qpel16_mc00_neon, export=1
261         mov             r3,  #16
262         .endfunc
263
264         pixfunc  avg_ pixels16,, 1
265
266 function ff_put_h264_qpel8_mc00_neon, export=1
267         mov             r3,  #8
268         .endfunc
269
270         pixfunc  put_ pixels8
271         pixfunc2 put_ pixels8_x2,   _no_rnd, vhadd.u8
272         pixfunc2 put_ pixels8_y2,   _no_rnd, vhadd.u8
273         pixfunc2 put_ pixels8_xy2,  _no_rnd, vshrn.u16, 1
274
275 function ff_avg_h264_qpel8_mc00_neon, export=1
276         mov             r3,  #8
277         .endfunc
278
279         pixfunc  avg_ pixels8,, 1
280
281 function ff_put_pixels_clamped_neon, export=1
282         vld1.64         {d16-d19}, [r0,:128]!
283         vqmovun.s16     d0, q8
284         vld1.64         {d20-d23}, [r0,:128]!
285         vqmovun.s16     d1, q9
286         vld1.64         {d24-d27}, [r0,:128]!
287         vqmovun.s16     d2, q10
288         vld1.64         {d28-d31}, [r0,:128]!
289         vqmovun.s16     d3, q11
290         vst1.64         {d0},      [r1,:64], r2
291         vqmovun.s16     d4, q12
292         vst1.64         {d1},      [r1,:64], r2
293         vqmovun.s16     d5, q13
294         vst1.64         {d2},      [r1,:64], r2
295         vqmovun.s16     d6, q14
296         vst1.64         {d3},      [r1,:64], r2
297         vqmovun.s16     d7, q15
298         vst1.64         {d4},      [r1,:64], r2
299         vst1.64         {d5},      [r1,:64], r2
300         vst1.64         {d6},      [r1,:64], r2
301         vst1.64         {d7},      [r1,:64], r2
302         bx              lr
303         .endfunc
304
305 function ff_put_signed_pixels_clamped_neon, export=1
306         vmov.u8         d31, #128
307         vld1.64         {d16-d17}, [r0,:128]!
308         vqmovn.s16      d0, q8
309         vld1.64         {d18-d19}, [r0,:128]!
310         vqmovn.s16      d1, q9
311         vld1.64         {d16-d17}, [r0,:128]!
312         vqmovn.s16      d2, q8
313         vld1.64         {d18-d19}, [r0,:128]!
314         vadd.u8         d0, d0, d31
315         vld1.64         {d20-d21}, [r0,:128]!
316         vadd.u8         d1, d1, d31
317         vld1.64         {d22-d23}, [r0,:128]!
318         vadd.u8         d2, d2, d31
319         vst1.64         {d0},      [r1,:64], r2
320         vqmovn.s16      d3, q9
321         vst1.64         {d1},      [r1,:64], r2
322         vqmovn.s16      d4, q10
323         vst1.64         {d2},      [r1,:64], r2
324         vqmovn.s16      d5, q11
325         vld1.64         {d24-d25}, [r0,:128]!
326         vadd.u8         d3, d3, d31
327         vld1.64         {d26-d27}, [r0,:128]!
328         vadd.u8         d4, d4, d31
329         vadd.u8         d5, d5, d31
330         vst1.64         {d3},      [r1,:64], r2
331         vqmovn.s16      d6, q12
332         vst1.64         {d4},      [r1,:64], r2
333         vqmovn.s16      d7, q13
334         vst1.64         {d5},      [r1,:64], r2
335         vadd.u8         d6, d6, d31
336         vadd.u8         d7, d7, d31
337         vst1.64         {d6},      [r1,:64], r2
338         vst1.64         {d7},      [r1,:64], r2
339         bx              lr
340         .endfunc
341
342 function ff_add_pixels_clamped_neon, export=1
343         mov             r3, r1
344         vld1.64         {d16},   [r1,:64], r2
345         vld1.64         {d0-d1}, [r0,:128]!
346         vaddw.u8        q0, q0, d16
347         vld1.64         {d17},   [r1,:64], r2
348         vld1.64         {d2-d3}, [r0,:128]!
349         vqmovun.s16     d0, q0
350         vld1.64         {d18},   [r1,:64], r2
351         vaddw.u8        q1, q1, d17
352         vld1.64         {d4-d5}, [r0,:128]!
353         vaddw.u8        q2, q2, d18
354         vst1.64         {d0},    [r3,:64], r2
355         vqmovun.s16     d2, q1
356         vld1.64         {d19},   [r1,:64], r2
357         vld1.64         {d6-d7}, [r0,:128]!
358         vaddw.u8        q3, q3, d19
359         vqmovun.s16     d4, q2
360         vst1.64         {d2},    [r3,:64], r2
361         vld1.64         {d16},   [r1,:64], r2
362         vqmovun.s16     d6, q3
363         vld1.64         {d0-d1}, [r0,:128]!
364         vaddw.u8        q0, q0, d16
365         vst1.64         {d4},    [r3,:64], r2
366         vld1.64         {d17},   [r1,:64], r2
367         vld1.64         {d2-d3}, [r0,:128]!
368         vaddw.u8        q1, q1, d17
369         vst1.64         {d6},    [r3,:64], r2
370         vqmovun.s16     d0, q0
371         vld1.64         {d18},   [r1,:64], r2
372         vld1.64         {d4-d5}, [r0,:128]!
373         vaddw.u8        q2, q2, d18
374         vst1.64         {d0},    [r3,:64], r2
375         vqmovun.s16     d2, q1
376         vld1.64         {d19},   [r1,:64], r2
377         vqmovun.s16     d4, q2
378         vld1.64         {d6-d7}, [r0,:128]!
379         vaddw.u8        q3, q3, d19
380         vst1.64         {d2},    [r3,:64], r2
381         vqmovun.s16     d6, q3
382         vst1.64         {d4},    [r3,:64], r2
383         vst1.64         {d6},    [r3,:64], r2
384         bx              lr
385         .endfunc
386
387 function ff_float_to_int16_neon, export=1
388         subs            r2,  r2,  #8
389         vld1.64         {d0-d1},  [r1,:128]!
390         vcvt.s32.f32    q8,  q0,  #16
391         vld1.64         {d2-d3},  [r1,:128]!
392         vcvt.s32.f32    q9,  q1,  #16
393         beq             3f
394         bics            ip,  r2,  #15
395         beq             2f
396 1:      subs            ip,  ip,  #16
397         vshrn.s32       d4,  q8,  #16
398         vld1.64         {d0-d1},  [r1,:128]!
399         vcvt.s32.f32    q0,  q0,  #16
400         vshrn.s32       d5,  q9,  #16
401         vld1.64         {d2-d3},  [r1,:128]!
402         vcvt.s32.f32    q1,  q1,  #16
403         vshrn.s32       d6,  q0,  #16
404         vst1.64         {d4-d5},  [r0,:128]!
405         vshrn.s32       d7,  q1,  #16
406         vld1.64         {d16-d17},[r1,:128]!
407         vcvt.s32.f32    q8,  q8,  #16
408         vld1.64         {d18-d19},[r1,:128]!
409         vcvt.s32.f32    q9,  q9,  #16
410         vst1.64         {d6-d7},  [r0,:128]!
411         bne             1b
412         ands            r2,  r2,  #15
413         beq             3f
414 2:      vld1.64         {d0-d1},  [r1,:128]!
415         vshrn.s32       d4,  q8,  #16
416         vcvt.s32.f32    q0,  q0,  #16
417         vld1.64         {d2-d3},  [r1,:128]!
418         vshrn.s32       d5,  q9,  #16
419         vcvt.s32.f32    q1,  q1,  #16
420         vshrn.s32       d6,  q0,  #16
421         vst1.64         {d4-d5},  [r0,:128]!
422         vshrn.s32       d7,  q1,  #16
423         vst1.64         {d6-d7},  [r0,:128]!
424         bx              lr
425 3:      vshrn.s32       d4,  q8,  #16
426         vshrn.s32       d5,  q9,  #16
427         vst1.64         {d4-d5},  [r0,:128]!
428         bx              lr
429         .endfunc
430
431 function ff_float_to_int16_interleave_neon, export=1
432         cmp             r3, #2
433         ldrlt           r1, [r1]
434         blt             ff_float_to_int16_neon
435         bne             4f
436
437         ldr             r3, [r1]
438         ldr             r1, [r1, #4]
439
440         subs            r2,  r2,  #8
441         vld1.64         {d0-d1},  [r3,:128]!
442         vcvt.s32.f32    q8,  q0,  #16
443         vld1.64         {d2-d3},  [r3,:128]!
444         vcvt.s32.f32    q9,  q1,  #16
445         vld1.64         {d20-d21},[r1,:128]!
446         vcvt.s32.f32    q10, q10, #16
447         vld1.64         {d22-d23},[r1,:128]!
448         vcvt.s32.f32    q11, q11, #16
449         beq             3f
450         bics            ip,  r2,  #15
451         beq             2f
452 1:      subs            ip,  ip,  #16
453         vld1.64         {d0-d1},  [r3,:128]!
454         vcvt.s32.f32    q0,  q0,  #16
455         vsri.32         q10, q8,  #16
456         vld1.64         {d2-d3},  [r3,:128]!
457         vcvt.s32.f32    q1,  q1,  #16
458         vld1.64         {d24-d25},[r1,:128]!
459         vcvt.s32.f32    q12, q12, #16
460         vld1.64         {d26-d27},[r1,:128]!
461         vsri.32         q11, q9,  #16
462         vst1.64         {d20-d21},[r0,:128]!
463         vcvt.s32.f32    q13, q13, #16
464         vst1.64         {d22-d23},[r0,:128]!
465         vsri.32         q12, q0,  #16
466         vld1.64         {d16-d17},[r3,:128]!
467         vsri.32         q13, q1,  #16
468         vst1.64         {d24-d25},[r0,:128]!
469         vcvt.s32.f32    q8,  q8,  #16
470         vld1.64         {d18-d19},[r3,:128]!
471         vcvt.s32.f32    q9,  q9,  #16
472         vld1.64         {d20-d21},[r1,:128]!
473         vcvt.s32.f32    q10, q10, #16
474         vld1.64         {d22-d23},[r1,:128]!
475         vcvt.s32.f32    q11, q11, #16
476         vst1.64         {d26-d27},[r0,:128]!
477         bne             1b
478         ands            r2,  r2,  #15
479         beq             3f
480 2:      vsri.32         q10, q8,  #16
481         vld1.64         {d0-d1},  [r3,:128]!
482         vcvt.s32.f32    q0,  q0,  #16
483         vld1.64         {d2-d3},  [r3,:128]!
484         vcvt.s32.f32    q1,  q1,  #16
485         vld1.64         {d24-d25},[r1,:128]!
486         vcvt.s32.f32    q12, q12, #16
487         vsri.32         q11, q9,  #16
488         vld1.64         {d26-d27},[r1,:128]!
489         vcvt.s32.f32    q13, q13, #16
490         vst1.64         {d20-d21},[r0,:128]!
491         vsri.32         q12, q0,  #16
492         vst1.64         {d22-d23},[r0,:128]!
493         vsri.32         q13, q1,  #16
494         vst1.64         {d24-d27},[r0,:128]!
495         bx              lr
496 3:      vsri.32         q10, q8,  #16
497         vsri.32         q11, q9,  #16
498         vst1.64         {d20-d23},[r0,:128]!
499         bx              lr
500
501 4:      push            {r4-r8,lr}
502         cmp             r3,  #4
503         lsl             ip,  r3,  #1
504         blt             4f
505
506         @ 4 channels
507 5:      ldmia           r1!, {r4-r7}
508         mov             lr,  r2
509         mov             r8,  r0
510         vld1.64         {d16-d17},[r4,:128]!
511         vcvt.s32.f32    q8,  q8,  #16
512         vld1.64         {d18-d19},[r5,:128]!
513         vcvt.s32.f32    q9,  q9,  #16
514         vld1.64         {d20-d21},[r6,:128]!
515         vcvt.s32.f32    q10, q10, #16
516         vld1.64         {d22-d23},[r7,:128]!
517         vcvt.s32.f32    q11, q11, #16
518 6:      subs            lr,  lr,  #8
519         vld1.64         {d0-d1},  [r4,:128]!
520         vcvt.s32.f32    q0,  q0,  #16
521         vsri.32         q9,  q8,  #16
522         vld1.64         {d2-d3},  [r5,:128]!
523         vcvt.s32.f32    q1,  q1,  #16
524         vsri.32         q11, q10, #16
525         vld1.64         {d4-d5},  [r6,:128]!
526         vcvt.s32.f32    q2,  q2,  #16
527         vzip.32         d18, d22
528         vld1.64         {d6-d7},  [r7,:128]!
529         vcvt.s32.f32    q3,  q3,  #16
530         vzip.32         d19, d23
531         vst1.64         {d18},    [r8], ip
532         vsri.32         q1,  q0,  #16
533         vst1.64         {d22},    [r8], ip
534         vsri.32         q3,  q2,  #16
535         vst1.64         {d19},    [r8], ip
536         vzip.32         d2,  d6
537         vst1.64         {d23},    [r8], ip
538         vzip.32         d3,  d7
539         beq             7f
540         vld1.64         {d16-d17},[r4,:128]!
541         vcvt.s32.f32    q8,  q8,  #16
542         vst1.64         {d2},     [r8], ip
543         vld1.64         {d18-d19},[r5,:128]!
544         vcvt.s32.f32    q9,  q9,  #16
545         vst1.64         {d6},     [r8], ip
546         vld1.64         {d20-d21},[r6,:128]!
547         vcvt.s32.f32    q10, q10, #16
548         vst1.64         {d3},     [r8], ip
549         vld1.64         {d22-d23},[r7,:128]!
550         vcvt.s32.f32    q11, q11, #16
551         vst1.64         {d7},     [r8], ip
552         b               6b
553 7:      vst1.64         {d2},     [r8], ip
554         vst1.64         {d6},     [r8], ip
555         vst1.64         {d3},     [r8], ip
556         vst1.64         {d7},     [r8], ip
557         subs            r3,  r3,  #4
558         popeq           {r4-r8,pc}
559         cmp             r3,  #4
560         add             r0,  r0,  #8
561         bge             5b
562
563         @ 2 channels
564 4:      cmp             r3,  #2
565         blt             4f
566         ldmia           r1!, {r4-r5}
567         mov             lr,  r2
568         mov             r8,  r0
569         tst             lr,  #8
570         vld1.64         {d16-d17},[r4,:128]!
571         vcvt.s32.f32    q8,  q8,  #16
572         vld1.64         {d18-d19},[r5,:128]!
573         vcvt.s32.f32    q9,  q9,  #16
574         vld1.64         {d20-d21},[r4,:128]!
575         vcvt.s32.f32    q10, q10, #16
576         vld1.64         {d22-d23},[r5,:128]!
577         vcvt.s32.f32    q11, q11, #16
578         beq             6f
579         subs            lr,  lr,  #8
580         beq             7f
581         vsri.32         d18, d16, #16
582         vsri.32         d19, d17, #16
583         vld1.64         {d16-d17},[r4,:128]!
584         vcvt.s32.f32    q8,  q8,  #16
585         vst1.32         {d18[0]}, [r8], ip
586         vsri.32         d22, d20, #16
587         vst1.32         {d18[1]}, [r8], ip
588         vsri.32         d23, d21, #16
589         vst1.32         {d19[0]}, [r8], ip
590         vst1.32         {d19[1]}, [r8], ip
591         vld1.64         {d18-d19},[r5,:128]!
592         vcvt.s32.f32    q9,  q9,  #16
593         vst1.32         {d22[0]}, [r8], ip
594         vst1.32         {d22[1]}, [r8], ip
595         vld1.64         {d20-d21},[r4,:128]!
596         vcvt.s32.f32    q10, q10, #16
597         vst1.32         {d23[0]}, [r8], ip
598         vst1.32         {d23[1]}, [r8], ip
599         vld1.64         {d22-d23},[r5,:128]!
600         vcvt.s32.f32    q11, q11, #16
601 6:      subs            lr,  lr,  #16
602         vld1.64         {d0-d1},  [r4,:128]!
603         vcvt.s32.f32    q0,  q0,  #16
604         vsri.32         d18, d16, #16
605         vld1.64         {d2-d3},  [r5,:128]!
606         vcvt.s32.f32    q1,  q1,  #16
607         vsri.32         d19, d17, #16
608         vld1.64         {d4-d5},  [r4,:128]!
609         vcvt.s32.f32    q2,  q2,  #16
610         vld1.64         {d6-d7},  [r5,:128]!
611         vcvt.s32.f32    q3,  q3,  #16
612         vst1.32         {d18[0]}, [r8], ip
613         vsri.32         d22, d20, #16
614         vst1.32         {d18[1]}, [r8], ip
615         vsri.32         d23, d21, #16
616         vst1.32         {d19[0]}, [r8], ip
617         vsri.32         d2,  d0,  #16
618         vst1.32         {d19[1]}, [r8], ip
619         vsri.32         d3,  d1,  #16
620         vst1.32         {d22[0]}, [r8], ip
621         vsri.32         d6,  d4,  #16
622         vst1.32         {d22[1]}, [r8], ip
623         vsri.32         d7,  d5,  #16
624         vst1.32         {d23[0]}, [r8], ip
625         vst1.32         {d23[1]}, [r8], ip
626         beq             6f
627         vld1.64         {d16-d17},[r4,:128]!
628         vcvt.s32.f32    q8,  q8,  #16
629         vst1.32         {d2[0]},  [r8], ip
630         vst1.32         {d2[1]},  [r8], ip
631         vld1.64         {d18-d19},[r5,:128]!
632         vcvt.s32.f32    q9,  q9,  #16
633         vst1.32         {d3[0]},  [r8], ip
634         vst1.32         {d3[1]},  [r8], ip
635         vld1.64         {d20-d21},[r4,:128]!
636         vcvt.s32.f32    q10, q10, #16
637         vst1.32         {d6[0]},  [r8], ip
638         vst1.32         {d6[1]},  [r8], ip
639         vld1.64         {d22-d23},[r5,:128]!
640         vcvt.s32.f32    q11, q11, #16
641         vst1.32         {d7[0]},  [r8], ip
642         vst1.32         {d7[1]},  [r8], ip
643         bgt             6b
644 6:      vst1.32         {d2[0]},  [r8], ip
645         vst1.32         {d2[1]},  [r8], ip
646         vst1.32         {d3[0]},  [r8], ip
647         vst1.32         {d3[1]},  [r8], ip
648         vst1.32         {d6[0]},  [r8], ip
649         vst1.32         {d6[1]},  [r8], ip
650         vst1.32         {d7[0]},  [r8], ip
651         vst1.32         {d7[1]},  [r8], ip
652         b               8f
653 7:      vsri.32         d18, d16, #16
654         vsri.32         d19, d17, #16
655         vst1.32         {d18[0]}, [r8], ip
656         vsri.32         d22, d20, #16
657         vst1.32         {d18[1]}, [r8], ip
658         vsri.32         d23, d21, #16
659         vst1.32         {d19[0]}, [r8], ip
660         vst1.32         {d19[1]}, [r8], ip
661         vst1.32         {d22[0]}, [r8], ip
662         vst1.32         {d22[1]}, [r8], ip
663         vst1.32         {d23[0]}, [r8], ip
664         vst1.32         {d23[1]}, [r8], ip
665 8:      subs            r3,  r3,  #2
666         add             r0,  r0,  #4
667         popeq           {r4-r8,pc}
668
669         @ 1 channel
670 4:      ldr             r4,  [r1],#4
671         tst             r2,  #8
672         mov             lr,  r2
673         mov             r5,  r0
674         vld1.64         {d0-d1},  [r4,:128]!
675         vcvt.s32.f32    q0,  q0,  #16
676         vld1.64         {d2-d3},  [r4,:128]!
677         vcvt.s32.f32    q1,  q1,  #16
678         bne             8f
679 6:      subs            lr,  lr,  #16
680         vld1.64         {d4-d5},  [r4,:128]!
681         vcvt.s32.f32    q2,  q2,  #16
682         vld1.64         {d6-d7},  [r4,:128]!
683         vcvt.s32.f32    q3,  q3,  #16
684         vst1.16         {d0[1]},  [r5,:16], ip
685         vst1.16         {d0[3]},  [r5,:16], ip
686         vst1.16         {d1[1]},  [r5,:16], ip
687         vst1.16         {d1[3]},  [r5,:16], ip
688         vst1.16         {d2[1]},  [r5,:16], ip
689         vst1.16         {d2[3]},  [r5,:16], ip
690         vst1.16         {d3[1]},  [r5,:16], ip
691         vst1.16         {d3[3]},  [r5,:16], ip
692         beq             7f
693         vld1.64         {d0-d1},  [r4,:128]!
694         vcvt.s32.f32    q0,  q0,  #16
695         vld1.64         {d2-d3},  [r4,:128]!
696         vcvt.s32.f32    q1,  q1,  #16
697 7:      vst1.16         {d4[1]},  [r5,:16], ip
698         vst1.16         {d4[3]},  [r5,:16], ip
699         vst1.16         {d5[1]},  [r5,:16], ip
700         vst1.16         {d5[3]},  [r5,:16], ip
701         vst1.16         {d6[1]},  [r5,:16], ip
702         vst1.16         {d6[3]},  [r5,:16], ip
703         vst1.16         {d7[1]},  [r5,:16], ip
704         vst1.16         {d7[3]},  [r5,:16], ip
705         bgt             6b
706         pop             {r4-r8,pc}
707 8:      subs            lr,  lr,  #8
708         vst1.16         {d0[1]},  [r5,:16], ip
709         vst1.16         {d0[3]},  [r5,:16], ip
710         vst1.16         {d1[1]},  [r5,:16], ip
711         vst1.16         {d1[3]},  [r5,:16], ip
712         vst1.16         {d2[1]},  [r5,:16], ip
713         vst1.16         {d2[3]},  [r5,:16], ip
714         vst1.16         {d3[1]},  [r5,:16], ip
715         vst1.16         {d3[3]},  [r5,:16], ip
716         popeq           {r4-r8,pc}
717         vld1.64         {d0-d1},  [r4,:128]!
718         vcvt.s32.f32    q0,  q0,  #16
719         vld1.64         {d2-d3},  [r4,:128]!
720         vcvt.s32.f32    q1,  q1,  #16
721         b               6b
722         .endfunc
723
724 function ff_vector_fmul_neon, export=1
725         mov             r3,  r0
726         subs            r2,  r2,  #8
727         vld1.64         {d0-d3},  [r0,:128]!
728         vld1.64         {d4-d7},  [r1,:128]!
729         vmul.f32        q8,  q0,  q2
730         vmul.f32        q9,  q1,  q3
731         beq             3f
732         bics            ip,  r2,  #15
733         beq             2f
734 1:      subs            ip,  ip,  #16
735         vld1.64         {d0-d1},  [r0,:128]!
736         vld1.64         {d4-d5},  [r1,:128]!
737         vmul.f32        q10, q0,  q2
738         vld1.64         {d2-d3},  [r0,:128]!
739         vld1.64         {d6-d7},  [r1,:128]!
740         vmul.f32        q11, q1,  q3
741         vst1.64         {d16-d19},[r3,:128]!
742         vld1.64         {d0-d1},  [r0,:128]!
743         vld1.64         {d4-d5},  [r1,:128]!
744         vmul.f32        q8,  q0,  q2
745         vld1.64         {d2-d3},  [r0,:128]!
746         vld1.64         {d6-d7},  [r1,:128]!
747         vmul.f32        q9,  q1,  q3
748         vst1.64         {d20-d23},[r3,:128]!
749         bne             1b
750         ands            r2,  r2,  #15
751         beq             3f
752 2:      vld1.64         {d0-d1},  [r0,:128]!
753         vld1.64         {d4-d5},  [r1,:128]!
754         vst1.64         {d16-d17},[r3,:128]!
755         vmul.f32        q8,  q0,  q2
756         vld1.64         {d2-d3},  [r0,:128]!
757         vld1.64         {d6-d7},  [r1,:128]!
758         vst1.64         {d18-d19},[r3,:128]!
759         vmul.f32        q9,  q1,  q3
760 3:      vst1.64         {d16-d19},[r3,:128]!
761         bx              lr
762         .endfunc
763
764 function ff_vector_fmul_window_neon, export=1
765 VFP     vdup.32         q8,  d0[0]
766 NOVFP   vld1.32         {d16[],d17[]}, [sp,:32]
767         push            {r4,r5,lr}
768 VFP     ldr             lr,  [sp, #12]
769 NOVFP   ldr             lr,  [sp, #16]
770         sub             r2,  r2,  #8
771         sub             r5,  lr,  #2
772         add             r2,  r2,  r5, lsl #2
773         add             r4,  r3,  r5, lsl #3
774         add             ip,  r0,  r5, lsl #3
775         mov             r5,  #-16
776         vld1.64         {d0,d1},  [r1,:128]!
777         vld1.64         {d2,d3},  [r2,:128], r5
778         vld1.64         {d4,d5},  [r3,:128]!
779         vld1.64         {d6,d7},  [r4,:128], r5
780 1:      subs            lr,  lr,  #4
781         vmov            q11, q8
782         vmla.f32        d22, d0,  d4
783         vmov            q10, q8
784         vmla.f32        d23, d1,  d5
785         vrev64.32       q3,  q3
786         vmla.f32        d20, d0,  d7
787         vrev64.32       q1,  q1
788         vmla.f32        d21, d1,  d6
789         beq             2f
790         vmla.f32        d22, d3,  d7
791         vld1.64         {d0,d1},  [r1,:128]!
792         vmla.f32        d23, d2,  d6
793         vld1.64         {d18,d19},[r2,:128], r5
794         vmls.f32        d20, d3,  d4
795         vld1.64         {d24,d25},[r3,:128]!
796         vmls.f32        d21, d2,  d5
797         vld1.64         {d6,d7},  [r4,:128], r5
798         vmov            q1,  q9
799         vrev64.32       q11, q11
800         vmov            q2,  q12
801         vswp            d22, d23
802         vst1.64         {d20,d21},[r0,:128]!
803         vst1.64         {d22,d23},[ip,:128], r5
804         b               1b
805 2:      vmla.f32        d22, d3,  d7
806         vmla.f32        d23, d2,  d6
807         vmls.f32        d20, d3,  d4
808         vmls.f32        d21, d2,  d5
809         vrev64.32       q11, q11
810         vswp            d22, d23
811         vst1.64         {d20,d21},[r0,:128]!
812         vst1.64         {d22,d23},[ip,:128], r5
813         pop             {r4,r5,pc}
814         .endfunc
815
816 #if CONFIG_VORBIS_DECODER
817 function ff_vorbis_inverse_coupling_neon, export=1
818         vmov.i32        q10, #1<<31
819         subs            r2,  r2,  #4
820         mov             r3,  r0
821         mov             r12, r1
822         beq             3f
823
824         vld1.32         {d24-d25},[r1,:128]!
825         vld1.32         {d22-d23},[r0,:128]!
826         vcle.s32        q8,  q12, #0
827         vand            q9,  q11, q10
828         veor            q12, q12, q9
829         vand            q2,  q12, q8
830         vbic            q3,  q12, q8
831         vadd.f32        q12, q11, q2
832         vsub.f32        q11, q11, q3
833 1:      vld1.32         {d2-d3},  [r1,:128]!
834         vld1.32         {d0-d1},  [r0,:128]!
835         vcle.s32        q8,  q1,  #0
836         vand            q9,  q0,  q10
837         veor            q1,  q1,  q9
838         vst1.32         {d24-d25},[r3, :128]!
839         vst1.32         {d22-d23},[r12,:128]!
840         vand            q2,  q1,  q8
841         vbic            q3,  q1,  q8
842         vadd.f32        q1,  q0,  q2
843         vsub.f32        q0,  q0,  q3
844         subs            r2,  r2,  #8
845         ble             2f
846         vld1.32         {d24-d25},[r1,:128]!
847         vld1.32         {d22-d23},[r0,:128]!
848         vcle.s32        q8,  q12, #0
849         vand            q9,  q11, q10
850         veor            q12, q12, q9
851         vst1.32         {d2-d3},  [r3, :128]!
852         vst1.32         {d0-d1},  [r12,:128]!
853         vand            q2,  q12, q8
854         vbic            q3,  q12, q8
855         vadd.f32        q12, q11, q2
856         vsub.f32        q11, q11, q3
857         b               1b
858
859 2:      vst1.32         {d2-d3},  [r3, :128]!
860         vst1.32         {d0-d1},  [r12,:128]!
861         bxlt            lr
862
863 3:      vld1.32         {d2-d3},  [r1,:128]
864         vld1.32         {d0-d1},  [r0,:128]
865         vcle.s32        q8,  q1,  #0
866         vand            q9,  q0,  q10
867         veor            q1,  q1,  q9
868         vand            q2,  q1,  q8
869         vbic            q3,  q1,  q8
870         vadd.f32        q1,  q0,  q2
871         vsub.f32        q0,  q0,  q3
872         vst1.32         {d2-d3},  [r0,:128]!
873         vst1.32         {d0-d1},  [r1,:128]!
874         bx              lr
875         .endfunc
876 #endif
877
878 function ff_vector_fmul_scalar_neon, export=1
879 VFP     len .req r2
880 NOVFP   len .req r3
881 VFP     vdup.32         q8,  d0[0]
882 NOVFP   vdup.32         q8,  r2
883         bics            r12, len, #15
884         beq             3f
885         vld1.32         {q0},[r1,:128]!
886         vld1.32         {q1},[r1,:128]!
887 1:      vmul.f32        q0,  q0,  q8
888         vld1.32         {q2},[r1,:128]!
889         vmul.f32        q1,  q1,  q8
890         vld1.32         {q3},[r1,:128]!
891         vmul.f32        q2,  q2,  q8
892         vst1.32         {q0},[r0,:128]!
893         vmul.f32        q3,  q3,  q8
894         vst1.32         {q1},[r0,:128]!
895         subs            r12, r12, #16
896         beq             2f
897         vld1.32         {q0},[r1,:128]!
898         vst1.32         {q2},[r0,:128]!
899         vld1.32         {q1},[r1,:128]!
900         vst1.32         {q3},[r0,:128]!
901         b               1b
902 2:      vst1.32         {q2},[r0,:128]!
903         vst1.32         {q3},[r0,:128]!
904         ands            len, len, #15
905         bxeq            lr
906 3:      vld1.32         {q0},[r1,:128]!
907         vmul.f32        q0,  q0,  q8
908         vst1.32         {q0},[r0,:128]!
909         subs            len, len, #4
910         bgt             3b
911         bx              lr
912         .unreq          len
913         .endfunc
914
915 function ff_vector_fmul_sv_scalar_2_neon, export=1
916 VFP     vdup.32         d16, d0[0]
917 NOVFP   vdup.32         d16, r3
918 NOVFP   ldr             r3,  [sp]
919         vld1.32         {d0},[r1,:64]!
920         vld1.32         {d1},[r1,:64]!
921 1:      subs            r3,  r3,  #4
922         vmul.f32        d4,  d0,  d16
923         vmul.f32        d5,  d1,  d16
924         ldr             r12, [r2], #4
925         vld1.32         {d2},[r12,:64]
926         ldr             r12, [r2], #4
927         vld1.32         {d3},[r12,:64]
928         vmul.f32        d4,  d4,  d2
929         vmul.f32        d5,  d5,  d3
930         beq             2f
931         vld1.32         {d0},[r1,:64]!
932         vld1.32         {d1},[r1,:64]!
933         vst1.32         {d4},[r0,:64]!
934         vst1.32         {d5},[r0,:64]!
935         b               1b
936 2:      vst1.32         {d4},[r0,:64]!
937         vst1.32         {d5},[r0,:64]!
938         bx              lr
939         .endfunc
940
941 function ff_vector_fmul_sv_scalar_4_neon, export=1
942 VFP     vdup.32         q10, d0[0]
943 NOVFP   vdup.32         q10, r3
944 NOVFP   ldr             r3,  [sp]
945         push            {lr}
946         bics            lr,  r3,  #7
947         beq             3f
948         vld1.32         {q0},[r1,:128]!
949         vld1.32         {q2},[r1,:128]!
950 1:      ldr             r12, [r2], #4
951         vld1.32         {q1},[r12,:128]
952         ldr             r12, [r2], #4
953         vld1.32         {q3},[r12,:128]
954         vmul.f32        q8,  q0,  q10
955         vmul.f32        q8,  q8,  q1
956         vmul.f32        q9,  q2,  q10
957         vmul.f32        q9,  q9,  q3
958         subs            lr,  lr,  #8
959         beq             2f
960         vld1.32         {q0},[r1,:128]!
961         vld1.32         {q2},[r1,:128]!
962         vst1.32         {q8},[r0,:128]!
963         vst1.32         {q9},[r0,:128]!
964         b               1b
965 2:      vst1.32         {q8},[r0,:128]!
966         vst1.32         {q9},[r0,:128]!
967         ands            r3,  r3,  #7
968         popeq           {pc}
969 3:      vld1.32         {q0},[r1,:128]!
970         ldr             r12, [r2], #4
971         vld1.32         {q1},[r12,:128]
972         vmul.f32        q0,  q0,  q10
973         vmul.f32        q0,  q0,  q1
974         vst1.32         {q0},[r0,:128]!
975         subs            r3,  r3,  #4
976         bgt             3b
977         pop             {pc}
978         .endfunc
979
980 function ff_sv_fmul_scalar_2_neon, export=1
981 VFP     len .req r2
982 NOVFP   len .req r3
983 VFP     vdup.32         q8,  d0[0]
984 NOVFP   vdup.32         q8,  r2
985         ldr             r12, [r1], #4
986         vld1.32         {d0},[r12,:64]
987         ldr             r12, [r1], #4
988         vld1.32         {d1},[r12,:64]
989 1:      vmul.f32        q1,  q0,  q8
990         subs            len, len, #4
991         beq             2f
992         ldr             r12, [r1], #4
993         vld1.32         {d0},[r12,:64]
994         ldr             r12, [r1], #4
995         vld1.32         {d1},[r12,:64]
996         vst1.32         {q1},[r0,:128]!
997         b               1b
998 2:      vst1.32         {q1},[r0,:128]!
999         bx              lr
1000         .unreq          len
1001         .endfunc
1002
1003 function ff_sv_fmul_scalar_4_neon, export=1
1004 VFP     len .req r2
1005 NOVFP   len .req r3
1006 VFP     vdup.32         q8,  d0[0]
1007 NOVFP   vdup.32         q8,  r2
1008 1:      ldr             r12, [r1], #4
1009         vld1.32         {q0},[r12,:128]
1010         vmul.f32        q0,  q0,  q8
1011         vst1.32         {q0},[r0,:128]!
1012         subs            len, len, #4
1013         bgt             1b
1014         bx              lr
1015         .unreq          len
1016         .endfunc
1017
1018 function ff_butterflies_float_neon, export=1
1019 1:      vld1.32         {q0},[r0,:128]
1020         vld1.32         {q1},[r1,:128]
1021         vsub.f32        q2,  q0,  q1
1022         vadd.f32        q1,  q0,  q1
1023         vst1.32         {q2},[r1,:128]!
1024         vst1.32         {q1},[r0,:128]!
1025         subs            r2,  r2,  #4
1026         bgt             1b
1027         bx              lr
1028         .endfunc
1029
1030 function ff_scalarproduct_float_neon, export=1
1031         vmov.f32        q2,  #0.0
1032 1:      vld1.32         {q0},[r0,:128]!
1033         vld1.32         {q1},[r1,:128]!
1034         vmla.f32        q2,  q0,  q1
1035         subs            r2,  r2,  #4
1036         bgt             1b
1037         vadd.f32        d0,  d4,  d5
1038         vpadd.f32       d0,  d0,  d0
1039 NOVFP   vmov.32         r0,  d0[0]
1040         bx              lr
1041         .endfunc
1042
1043 function ff_int32_to_float_fmul_scalar_neon, export=1
1044 VFP     vdup.32         q0,  d0[0]
1045 VFP     len     .req    r2
1046 NOVFP   vdup.32         q0,  r2
1047 NOVFP   len     .req    r3
1048
1049         vld1.32         {q1},[r1,:128]!
1050         vcvt.f32.s32    q3,  q1
1051         vld1.32         {q2},[r1,:128]!
1052         vcvt.f32.s32    q8,  q2
1053 1:      subs            len, len, #8
1054         pld             [r1, #16]
1055         vmul.f32        q9,  q3,  q0
1056         vmul.f32        q10, q8,  q0
1057         beq             2f
1058         vld1.32         {q1},[r1,:128]!
1059         vcvt.f32.s32    q3,  q1
1060         vld1.32         {q2},[r1,:128]!
1061         vcvt.f32.s32    q8,  q2
1062         vst1.32         {q9}, [r0,:128]!
1063         vst1.32         {q10},[r0,:128]!
1064         b               1b
1065 2:      vst1.32         {q9}, [r0,:128]!
1066         vst1.32         {q10},[r0,:128]!
1067         bx              lr
1068         .unreq  len
1069         .endfunc
1070
1071 function ff_vector_fmul_reverse_neon, export=1
1072         add             r2,  r2,  r3,  lsl #2
1073         sub             r2,  r2,  #32
1074         mov             r12, #-32
1075         vld1.32         {q0-q1},  [r1,:128]!
1076         vld1.32         {q2-q3},  [r2,:128], r12
1077 1:      pld             [r1, #32]
1078         vrev64.32       q3,  q3
1079         vmul.f32        d16, d0,  d7
1080         vmul.f32        d17, d1,  d6
1081         pld             [r2, #-32]
1082         vrev64.32       q2,  q2
1083         vmul.f32        d18, d2,  d5
1084         vmul.f32        d19, d3,  d4
1085         subs            r3,  r3,  #8
1086         beq             2f
1087         vld1.32         {q0-q1},  [r1,:128]!
1088         vld1.32         {q2-q3},  [r2,:128], r12
1089         vst1.32         {q8-q9},  [r0,:128]!
1090         b               1b
1091 2:      vst1.32         {q8-q9},  [r0,:128]!
1092         bx              lr
1093         .endfunc
1094
1095 function ff_vector_fmul_add_neon, export=1
1096         ldr             r12, [sp]
1097         vld1.32         {q0-q1},  [r1,:128]!
1098         vld1.32         {q8-q9},  [r2,:128]!
1099         vld1.32         {q2-q3},  [r3,:128]!
1100         vmul.f32        q10, q0,  q8
1101         vmul.f32        q11, q1,  q9
1102 1:      vadd.f32        q12, q2,  q10
1103         vadd.f32        q13, q3,  q11
1104         pld             [r1, #16]
1105         pld             [r2, #16]
1106         pld             [r3, #16]
1107         subs            r12, r12, #8
1108         beq             2f
1109         vld1.32         {q0},     [r1,:128]!
1110         vld1.32         {q8},     [r2,:128]!
1111         vmul.f32        q10, q0,  q8
1112         vld1.32         {q1},     [r1,:128]!
1113         vld1.32         {q9},     [r2,:128]!
1114         vmul.f32        q11, q1,  q9
1115         vld1.32         {q2-q3},  [r3,:128]!
1116         vst1.32         {q12-q13},[r0,:128]!
1117         b               1b
1118 2:      vst1.32         {q12-q13},[r0,:128]!
1119         bx              lr
1120         .endfunc
1121
1122 function ff_vector_clipf_neon, export=1
1123 VFP     vdup.32         q1,  d0[1]
1124 VFP     vdup.32         q0,  d0[0]
1125 NOVFP   vdup.32         q0,  r2
1126 NOVFP   vdup.32         q1,  r3
1127 NOVFP   ldr             r2,  [sp]
1128         vld1.f32        {q2},[r1,:128]!
1129         vmin.f32        q10, q2,  q1
1130         vld1.f32        {q3},[r1,:128]!
1131         vmin.f32        q11, q3,  q1
1132 1:      vmax.f32        q8,  q10, q0
1133         vmax.f32        q9,  q11, q0
1134         subs            r2,  r2,  #8
1135         beq             2f
1136         vld1.f32        {q2},[r1,:128]!
1137         vmin.f32        q10, q2,  q1
1138         vld1.f32        {q3},[r1,:128]!
1139         vmin.f32        q11, q3,  q1
1140         vst1.f32        {q8},[r0,:128]!
1141         vst1.f32        {q9},[r0,:128]!
1142         b               1b
1143 2:      vst1.f32        {q8},[r0,:128]!
1144         vst1.f32        {q9},[r0,:128]!
1145         bx              lr
1146         .endfunc