dsputil: move vector_fmul_scalar() to AVFloatDSPContext in libavutil
[ffmpeg.git] / libavcodec / arm / dsputil_neon.S
1 /*
2  * ARM NEON optimised DSP functions
3  * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
4  *
5  * This file is part of Libav.
6  *
7  * Libav is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * Libav is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with Libav; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 #include "config.h"
23 #include "libavutil/arm/asm.S"
24
25 function ff_clear_block_neon, export=1
26         vmov.i16        q0,  #0
27         .rept           8
28         vst1.16         {q0}, [r0,:128]!
29         .endr
30         bx              lr
31 endfunc
32
33 function ff_clear_blocks_neon, export=1
34         vmov.i16        q0,  #0
35         .rept           8*6
36         vst1.16         {q0}, [r0,:128]!
37         .endr
38         bx              lr
39 endfunc
40
41 .macro  pixels16        rnd=1, avg=0
42   .if \avg
43         mov             r12, r0
44   .endif
45 1:      vld1.8          {q0},     [r1], r2
46         vld1.8          {q1},     [r1], r2
47         vld1.8          {q2},     [r1], r2
48         pld             [r1, r2, lsl #2]
49         vld1.8          {q3},     [r1], r2
50         pld             [r1]
51         pld             [r1, r2]
52         pld             [r1, r2, lsl #1]
53   .if \avg
54         vld1.8          {q8},     [r12,:128], r2
55         vrhadd.u8       q0,  q0,  q8
56         vld1.8          {q9},     [r12,:128], r2
57         vrhadd.u8       q1,  q1,  q9
58         vld1.8          {q10},    [r12,:128], r2
59         vrhadd.u8       q2,  q2,  q10
60         vld1.8          {q11},    [r12,:128], r2
61         vrhadd.u8       q3,  q3,  q11
62   .endif
63         subs            r3,  r3,  #4
64         vst1.64         {q0},     [r0,:128], r2
65         vst1.64         {q1},     [r0,:128], r2
66         vst1.64         {q2},     [r0,:128], r2
67         vst1.64         {q3},     [r0,:128], r2
68         bne             1b
69         bx              lr
70 .endm
71
72 .macro  pixels16_x2     rnd=1, avg=0
73 1:      vld1.8          {d0-d2},  [r1], r2
74         vld1.8          {d4-d6},  [r1], r2
75         pld             [r1]
76         pld             [r1, r2]
77         subs            r3,  r3,  #2
78         vext.8          q1,  q0,  q1,  #1
79         avg             q0,  q0,  q1
80         vext.8          q3,  q2,  q3,  #1
81         avg             q2,  q2,  q3
82   .if \avg
83         vld1.8          {q1},     [r0,:128], r2
84         vld1.8          {q3},     [r0,:128]
85         vrhadd.u8       q0,  q0,  q1
86         vrhadd.u8       q2,  q2,  q3
87         sub             r0,  r0,  r2
88   .endif
89         vst1.8          {q0},     [r0,:128], r2
90         vst1.8          {q2},     [r0,:128], r2
91         bne             1b
92         bx              lr
93 .endm
94
95 .macro  pixels16_y2     rnd=1, avg=0
96         sub             r3,  r3,  #2
97         vld1.8          {q0},     [r1], r2
98         vld1.8          {q1},     [r1], r2
99 1:      subs            r3,  r3,  #2
100         avg             q2,  q0,  q1
101         vld1.8          {q0},     [r1], r2
102         avg             q3,  q0,  q1
103         vld1.8          {q1},     [r1], r2
104         pld             [r1]
105         pld             [r1, r2]
106   .if \avg
107         vld1.8          {q8},     [r0,:128], r2
108         vld1.8          {q9},     [r0,:128]
109         vrhadd.u8       q2,  q2,  q8
110         vrhadd.u8       q3,  q3,  q9
111         sub             r0,  r0,  r2
112   .endif
113         vst1.8          {q2},     [r0,:128], r2
114         vst1.8          {q3},     [r0,:128], r2
115         bne             1b
116
117         avg             q2,  q0,  q1
118         vld1.8          {q0},     [r1], r2
119         avg             q3,  q0,  q1
120   .if \avg
121         vld1.8          {q8},     [r0,:128], r2
122         vld1.8          {q9},     [r0,:128]
123         vrhadd.u8       q2,  q2,  q8
124         vrhadd.u8       q3,  q3,  q9
125         sub             r0,  r0,  r2
126   .endif
127         vst1.8          {q2},     [r0,:128], r2
128         vst1.8          {q3},     [r0,:128], r2
129
130         bx              lr
131 .endm
132
133 .macro  pixels16_xy2    rnd=1, avg=0
134         sub             r3,  r3,  #2
135         vld1.8          {d0-d2},  [r1], r2
136         vld1.8          {d4-d6},  [r1], r2
137 NRND    vmov.i16        q13, #1
138         pld             [r1]
139         pld             [r1, r2]
140         vext.8          q1,  q0,  q1,  #1
141         vext.8          q3,  q2,  q3,  #1
142         vaddl.u8        q8,  d0,  d2
143         vaddl.u8        q10, d1,  d3
144         vaddl.u8        q9,  d4,  d6
145         vaddl.u8        q11, d5,  d7
146 1:      subs            r3,  r3,  #2
147         vld1.8          {d0-d2},  [r1], r2
148         vadd.u16        q12, q8,  q9
149         pld             [r1]
150 NRND    vadd.u16        q12, q12, q13
151         vext.8          q15, q0,  q1,  #1
152         vadd.u16        q1 , q10, q11
153         shrn            d28, q12, #2
154 NRND    vadd.u16        q1,  q1,  q13
155         shrn            d29, q1,  #2
156   .if \avg
157         vld1.8          {q8},     [r0,:128]
158         vrhadd.u8       q14, q14, q8
159   .endif
160         vaddl.u8        q8,  d0,  d30
161         vld1.8          {d2-d4},  [r1], r2
162         vaddl.u8        q10, d1,  d31
163         vst1.8          {q14},    [r0,:128], r2
164         vadd.u16        q12, q8,  q9
165         pld             [r1, r2]
166 NRND    vadd.u16        q12, q12, q13
167         vext.8          q2,  q1,  q2,  #1
168         vadd.u16        q0,  q10, q11
169         shrn            d30, q12, #2
170 NRND    vadd.u16        q0,  q0,  q13
171         shrn            d31, q0,  #2
172   .if \avg
173         vld1.8          {q9},     [r0,:128]
174         vrhadd.u8       q15, q15, q9
175   .endif
176         vaddl.u8        q9,  d2,  d4
177         vaddl.u8        q11, d3,  d5
178         vst1.8          {q15},    [r0,:128], r2
179         bgt             1b
180
181         vld1.8          {d0-d2},  [r1], r2
182         vadd.u16        q12, q8,  q9
183 NRND    vadd.u16        q12, q12, q13
184         vext.8          q15, q0,  q1,  #1
185         vadd.u16        q1 , q10, q11
186         shrn            d28, q12, #2
187 NRND    vadd.u16        q1,  q1,  q13
188         shrn            d29, q1,  #2
189   .if \avg
190         vld1.8          {q8},     [r0,:128]
191         vrhadd.u8       q14, q14, q8
192   .endif
193         vaddl.u8        q8,  d0,  d30
194         vaddl.u8        q10, d1,  d31
195         vst1.8          {q14},    [r0,:128], r2
196         vadd.u16        q12, q8,  q9
197 NRND    vadd.u16        q12, q12, q13
198         vadd.u16        q0,  q10, q11
199         shrn            d30, q12, #2
200 NRND    vadd.u16        q0,  q0,  q13
201         shrn            d31, q0,  #2
202   .if \avg
203         vld1.8          {q9},     [r0,:128]
204         vrhadd.u8       q15, q15, q9
205   .endif
206         vst1.8          {q15},    [r0,:128], r2
207
208         bx              lr
209 .endm
210
211 .macro  pixels8         rnd=1, avg=0
212 1:      vld1.8          {d0},     [r1], r2
213         vld1.8          {d1},     [r1], r2
214         vld1.8          {d2},     [r1], r2
215         pld             [r1, r2, lsl #2]
216         vld1.8          {d3},     [r1], r2
217         pld             [r1]
218         pld             [r1, r2]
219         pld             [r1, r2, lsl #1]
220   .if \avg
221         vld1.8          {d4},     [r0,:64], r2
222         vrhadd.u8       d0,  d0,  d4
223         vld1.8          {d5},     [r0,:64], r2
224         vrhadd.u8       d1,  d1,  d5
225         vld1.8          {d6},     [r0,:64], r2
226         vrhadd.u8       d2,  d2,  d6
227         vld1.8          {d7},     [r0,:64], r2
228         vrhadd.u8       d3,  d3,  d7
229         sub             r0,  r0,  r2,  lsl #2
230   .endif
231         subs            r3,  r3,  #4
232         vst1.8          {d0},     [r0,:64], r2
233         vst1.8          {d1},     [r0,:64], r2
234         vst1.8          {d2},     [r0,:64], r2
235         vst1.8          {d3},     [r0,:64], r2
236         bne             1b
237         bx              lr
238 .endm
239
240 .macro  pixels8_x2      rnd=1, avg=0
241 1:      vld1.8          {q0},     [r1], r2
242         vext.8          d1,  d0,  d1,  #1
243         vld1.8          {q1},     [r1], r2
244         vext.8          d3,  d2,  d3,  #1
245         pld             [r1]
246         pld             [r1, r2]
247         subs            r3,  r3,  #2
248         vswp            d1,  d2
249         avg             q0,  q0,  q1
250   .if \avg
251         vld1.8          {d4},     [r0,:64], r2
252         vld1.8          {d5},     [r0,:64]
253         vrhadd.u8       q0,  q0,  q2
254         sub             r0,  r0,  r2
255   .endif
256         vst1.8          {d0},     [r0,:64], r2
257         vst1.8          {d1},     [r0,:64], r2
258         bne             1b
259         bx              lr
260 .endm
261
262 .macro  pixels8_y2      rnd=1, avg=0
263         sub             r3,  r3,  #2
264         vld1.8          {d0},     [r1], r2
265         vld1.8          {d1},     [r1], r2
266 1:      subs            r3,  r3,  #2
267         avg             d4,  d0,  d1
268         vld1.8          {d0},     [r1], r2
269         avg             d5,  d0,  d1
270         vld1.8          {d1},     [r1], r2
271         pld             [r1]
272         pld             [r1, r2]
273   .if \avg
274         vld1.8          {d2},     [r0,:64], r2
275         vld1.8          {d3},     [r0,:64]
276         vrhadd.u8       q2,  q2,  q1
277         sub             r0,  r0,  r2
278   .endif
279         vst1.8          {d4},     [r0,:64], r2
280         vst1.8          {d5},     [r0,:64], r2
281         bne             1b
282
283         avg             d4,  d0,  d1
284         vld1.8          {d0},     [r1], r2
285         avg             d5,  d0,  d1
286   .if \avg
287         vld1.8          {d2},     [r0,:64], r2
288         vld1.8          {d3},     [r0,:64]
289         vrhadd.u8       q2,  q2,  q1
290         sub             r0,  r0,  r2
291   .endif
292         vst1.8          {d4},     [r0,:64], r2
293         vst1.8          {d5},     [r0,:64], r2
294
295         bx              lr
296 .endm
297
298 .macro  pixels8_xy2     rnd=1, avg=0
299         sub             r3,  r3,  #2
300         vld1.8          {q0},     [r1], r2
301         vld1.8          {q1},     [r1], r2
302 NRND    vmov.i16        q11, #1
303         pld             [r1]
304         pld             [r1, r2]
305         vext.8          d4,  d0,  d1,  #1
306         vext.8          d6,  d2,  d3,  #1
307         vaddl.u8        q8,  d0,  d4
308         vaddl.u8        q9,  d2,  d6
309 1:      subs            r3,  r3,  #2
310         vld1.8          {q0},     [r1], r2
311         pld             [r1]
312         vadd.u16        q10, q8,  q9
313         vext.8          d4,  d0,  d1,  #1
314 NRND    vadd.u16        q10, q10, q11
315         vaddl.u8        q8,  d0,  d4
316         shrn            d5,  q10, #2
317         vld1.8          {q1},     [r1], r2
318         vadd.u16        q10, q8,  q9
319         pld             [r1, r2]
320   .if \avg
321         vld1.8          {d7},     [r0,:64]
322         vrhadd.u8       d5,  d5,  d7
323   .endif
324 NRND    vadd.u16        q10, q10, q11
325         vst1.8          {d5},     [r0,:64], r2
326         shrn            d7,  q10, #2
327   .if \avg
328         vld1.8          {d5},     [r0,:64]
329         vrhadd.u8       d7,  d7,  d5
330   .endif
331         vext.8          d6,  d2,  d3,  #1
332         vaddl.u8        q9,  d2,  d6
333         vst1.8          {d7},     [r0,:64], r2
334         bgt             1b
335
336         vld1.8          {q0},     [r1], r2
337         vadd.u16        q10, q8,  q9
338         vext.8          d4,  d0,  d1,  #1
339 NRND    vadd.u16        q10, q10, q11
340         vaddl.u8        q8,  d0,  d4
341         shrn            d5,  q10, #2
342         vadd.u16        q10, q8,  q9
343   .if \avg
344         vld1.8          {d7},     [r0,:64]
345         vrhadd.u8       d5,  d5,  d7
346   .endif
347 NRND    vadd.u16        q10, q10, q11
348         vst1.8          {d5},     [r0,:64], r2
349         shrn            d7,  q10, #2
350   .if \avg
351         vld1.8          {d5},     [r0,:64]
352         vrhadd.u8       d7,  d7,  d5
353   .endif
354         vst1.8          {d7},     [r0,:64], r2
355
356         bx              lr
357 .endm
358
359 .macro  pixfunc         pfx, name, suf, rnd=1, avg=0
360   .if \rnd
361     .macro avg  rd, rn, rm
362         vrhadd.u8       \rd, \rn, \rm
363     .endm
364     .macro shrn rd, rn, rm
365         vrshrn.u16      \rd, \rn, \rm
366     .endm
367     .macro NRND insn:vararg
368     .endm
369   .else
370     .macro avg  rd, rn, rm
371         vhadd.u8        \rd, \rn, \rm
372     .endm
373     .macro shrn rd, rn, rm
374         vshrn.u16       \rd, \rn, \rm
375     .endm
376     .macro NRND insn:vararg
377         \insn
378     .endm
379   .endif
380 function ff_\pfx\name\suf\()_neon, export=1
381         \name           \rnd, \avg
382 endfunc
383         .purgem         avg
384         .purgem         shrn
385         .purgem         NRND
386 .endm
387
388 .macro  pixfunc2        pfx, name, avg=0
389         pixfunc         \pfx, \name,          rnd=1, avg=\avg
390         pixfunc         \pfx, \name, _no_rnd, rnd=0, avg=\avg
391 .endm
392
393 function ff_put_h264_qpel16_mc00_neon, export=1
394         mov             r3,  #16
395 endfunc
396
397         pixfunc         put_, pixels16,     avg=0
398         pixfunc2        put_, pixels16_x2,  avg=0
399         pixfunc2        put_, pixels16_y2,  avg=0
400         pixfunc2        put_, pixels16_xy2, avg=0
401
402 function ff_avg_h264_qpel16_mc00_neon, export=1
403         mov             r3,  #16
404 endfunc
405
406         pixfunc         avg_, pixels16,     avg=1
407         pixfunc2        avg_, pixels16_x2,  avg=1
408         pixfunc2        avg_, pixels16_y2,  avg=1
409         pixfunc2        avg_, pixels16_xy2, avg=1
410
411 function ff_put_h264_qpel8_mc00_neon, export=1
412         mov             r3,  #8
413 endfunc
414
415         pixfunc         put_, pixels8,     avg=0
416         pixfunc2        put_, pixels8_x2,  avg=0
417         pixfunc2        put_, pixels8_y2,  avg=0
418         pixfunc2        put_, pixels8_xy2, avg=0
419
420 function ff_avg_h264_qpel8_mc00_neon, export=1
421         mov             r3,  #8
422 endfunc
423
424         pixfunc         avg_, pixels8,     avg=1
425         pixfunc2        avg_, pixels8_x2,  avg=1
426         pixfunc2        avg_, pixels8_y2,  avg=1
427         pixfunc2        avg_, pixels8_xy2, avg=1
428
429 function ff_put_pixels_clamped_neon, export=1
430         vld1.16         {d16-d19}, [r0,:128]!
431         vqmovun.s16     d0, q8
432         vld1.16         {d20-d23}, [r0,:128]!
433         vqmovun.s16     d1, q9
434         vld1.16         {d24-d27}, [r0,:128]!
435         vqmovun.s16     d2, q10
436         vld1.16         {d28-d31}, [r0,:128]!
437         vqmovun.s16     d3, q11
438         vst1.8          {d0},      [r1,:64], r2
439         vqmovun.s16     d4, q12
440         vst1.8          {d1},      [r1,:64], r2
441         vqmovun.s16     d5, q13
442         vst1.8          {d2},      [r1,:64], r2
443         vqmovun.s16     d6, q14
444         vst1.8          {d3},      [r1,:64], r2
445         vqmovun.s16     d7, q15
446         vst1.8          {d4},      [r1,:64], r2
447         vst1.8          {d5},      [r1,:64], r2
448         vst1.8          {d6},      [r1,:64], r2
449         vst1.8          {d7},      [r1,:64], r2
450         bx              lr
451 endfunc
452
453 function ff_put_signed_pixels_clamped_neon, export=1
454         vmov.u8         d31, #128
455         vld1.16         {d16-d17}, [r0,:128]!
456         vqmovn.s16      d0, q8
457         vld1.16         {d18-d19}, [r0,:128]!
458         vqmovn.s16      d1, q9
459         vld1.16         {d16-d17}, [r0,:128]!
460         vqmovn.s16      d2, q8
461         vld1.16         {d18-d19}, [r0,:128]!
462         vadd.u8         d0, d0, d31
463         vld1.16         {d20-d21}, [r0,:128]!
464         vadd.u8         d1, d1, d31
465         vld1.16         {d22-d23}, [r0,:128]!
466         vadd.u8         d2, d2, d31
467         vst1.8          {d0},      [r1,:64], r2
468         vqmovn.s16      d3, q9
469         vst1.8          {d1},      [r1,:64], r2
470         vqmovn.s16      d4, q10
471         vst1.8          {d2},      [r1,:64], r2
472         vqmovn.s16      d5, q11
473         vld1.16         {d24-d25}, [r0,:128]!
474         vadd.u8         d3, d3, d31
475         vld1.16         {d26-d27}, [r0,:128]!
476         vadd.u8         d4, d4, d31
477         vadd.u8         d5, d5, d31
478         vst1.8          {d3},      [r1,:64], r2
479         vqmovn.s16      d6, q12
480         vst1.8          {d4},      [r1,:64], r2
481         vqmovn.s16      d7, q13
482         vst1.8          {d5},      [r1,:64], r2
483         vadd.u8         d6, d6, d31
484         vadd.u8         d7, d7, d31
485         vst1.8          {d6},      [r1,:64], r2
486         vst1.8          {d7},      [r1,:64], r2
487         bx              lr
488 endfunc
489
490 function ff_add_pixels_clamped_neon, export=1
491         mov             r3, r1
492         vld1.8          {d16},   [r1,:64], r2
493         vld1.16         {d0-d1}, [r0,:128]!
494         vaddw.u8        q0, q0, d16
495         vld1.8          {d17},   [r1,:64], r2
496         vld1.16         {d2-d3}, [r0,:128]!
497         vqmovun.s16     d0, q0
498         vld1.8          {d18},   [r1,:64], r2
499         vaddw.u8        q1, q1, d17
500         vld1.16         {d4-d5}, [r0,:128]!
501         vaddw.u8        q2, q2, d18
502         vst1.8          {d0},    [r3,:64], r2
503         vqmovun.s16     d2, q1
504         vld1.8          {d19},   [r1,:64], r2
505         vld1.16         {d6-d7}, [r0,:128]!
506         vaddw.u8        q3, q3, d19
507         vqmovun.s16     d4, q2
508         vst1.8          {d2},    [r3,:64], r2
509         vld1.8          {d16},   [r1,:64], r2
510         vqmovun.s16     d6, q3
511         vld1.16         {d0-d1}, [r0,:128]!
512         vaddw.u8        q0, q0, d16
513         vst1.8          {d4},    [r3,:64], r2
514         vld1.8          {d17},   [r1,:64], r2
515         vld1.16         {d2-d3}, [r0,:128]!
516         vaddw.u8        q1, q1, d17
517         vst1.8          {d6},    [r3,:64], r2
518         vqmovun.s16     d0, q0
519         vld1.8          {d18},   [r1,:64], r2
520         vld1.16         {d4-d5}, [r0,:128]!
521         vaddw.u8        q2, q2, d18
522         vst1.8          {d0},    [r3,:64], r2
523         vqmovun.s16     d2, q1
524         vld1.8          {d19},   [r1,:64], r2
525         vqmovun.s16     d4, q2
526         vld1.16         {d6-d7}, [r0,:128]!
527         vaddw.u8        q3, q3, d19
528         vst1.8          {d2},    [r3,:64], r2
529         vqmovun.s16     d6, q3
530         vst1.8          {d4},    [r3,:64], r2
531         vst1.8          {d6},    [r3,:64], r2
532         bx              lr
533 endfunc
534
535 function ff_vector_fmul_window_neon, export=1
536         push            {r4,r5,lr}
537         ldr             lr,  [sp, #12]
538         sub             r2,  r2,  #8
539         sub             r5,  lr,  #2
540         add             r2,  r2,  r5, lsl #2
541         add             r4,  r3,  r5, lsl #3
542         add             ip,  r0,  r5, lsl #3
543         mov             r5,  #-16
544         vld1.32         {d0,d1},  [r1,:128]!
545         vld1.32         {d2,d3},  [r2,:128], r5
546         vld1.32         {d4,d5},  [r3,:128]!
547         vld1.32         {d6,d7},  [r4,:128], r5
548 1:      subs            lr,  lr,  #4
549         vmul.f32        d22, d0,  d4
550         vrev64.32       q3,  q3
551         vmul.f32        d23, d1,  d5
552         vrev64.32       q1,  q1
553         vmul.f32        d20, d0,  d7
554         vmul.f32        d21, d1,  d6
555         beq             2f
556         vmla.f32        d22, d3,  d7
557         vld1.32         {d0,d1},  [r1,:128]!
558         vmla.f32        d23, d2,  d6
559         vld1.32         {d18,d19},[r2,:128], r5
560         vmls.f32        d20, d3,  d4
561         vld1.32         {d24,d25},[r3,:128]!
562         vmls.f32        d21, d2,  d5
563         vld1.32         {d6,d7},  [r4,:128], r5
564         vmov            q1,  q9
565         vrev64.32       q11, q11
566         vmov            q2,  q12
567         vswp            d22, d23
568         vst1.32         {d20,d21},[r0,:128]!
569         vst1.32         {d22,d23},[ip,:128], r5
570         b               1b
571 2:      vmla.f32        d22, d3,  d7
572         vmla.f32        d23, d2,  d6
573         vmls.f32        d20, d3,  d4
574         vmls.f32        d21, d2,  d5
575         vrev64.32       q11, q11
576         vswp            d22, d23
577         vst1.32         {d20,d21},[r0,:128]!
578         vst1.32         {d22,d23},[ip,:128], r5
579         pop             {r4,r5,pc}
580 endfunc
581
582 #if CONFIG_VORBIS_DECODER
583 function ff_vorbis_inverse_coupling_neon, export=1
584         vmov.i32        q10, #1<<31
585         subs            r2,  r2,  #4
586         mov             r3,  r0
587         mov             r12, r1
588         beq             3f
589
590         vld1.32         {d24-d25},[r1,:128]!
591         vld1.32         {d22-d23},[r0,:128]!
592         vcle.s32        q8,  q12, #0
593         vand            q9,  q11, q10
594         veor            q12, q12, q9
595         vand            q2,  q12, q8
596         vbic            q3,  q12, q8
597         vadd.f32        q12, q11, q2
598         vsub.f32        q11, q11, q3
599 1:      vld1.32         {d2-d3},  [r1,:128]!
600         vld1.32         {d0-d1},  [r0,:128]!
601         vcle.s32        q8,  q1,  #0
602         vand            q9,  q0,  q10
603         veor            q1,  q1,  q9
604         vst1.32         {d24-d25},[r3, :128]!
605         vst1.32         {d22-d23},[r12,:128]!
606         vand            q2,  q1,  q8
607         vbic            q3,  q1,  q8
608         vadd.f32        q1,  q0,  q2
609         vsub.f32        q0,  q0,  q3
610         subs            r2,  r2,  #8
611         ble             2f
612         vld1.32         {d24-d25},[r1,:128]!
613         vld1.32         {d22-d23},[r0,:128]!
614         vcle.s32        q8,  q12, #0
615         vand            q9,  q11, q10
616         veor            q12, q12, q9
617         vst1.32         {d2-d3},  [r3, :128]!
618         vst1.32         {d0-d1},  [r12,:128]!
619         vand            q2,  q12, q8
620         vbic            q3,  q12, q8
621         vadd.f32        q12, q11, q2
622         vsub.f32        q11, q11, q3
623         b               1b
624
625 2:      vst1.32         {d2-d3},  [r3, :128]!
626         vst1.32         {d0-d1},  [r12,:128]!
627         it              lt
628         bxlt            lr
629
630 3:      vld1.32         {d2-d3},  [r1,:128]
631         vld1.32         {d0-d1},  [r0,:128]
632         vcle.s32        q8,  q1,  #0
633         vand            q9,  q0,  q10
634         veor            q1,  q1,  q9
635         vand            q2,  q1,  q8
636         vbic            q3,  q1,  q8
637         vadd.f32        q1,  q0,  q2
638         vsub.f32        q0,  q0,  q3
639         vst1.32         {d2-d3},  [r0,:128]!
640         vst1.32         {d0-d1},  [r1,:128]!
641         bx              lr
642 endfunc
643 #endif
644
645 function ff_butterflies_float_neon, export=1
646 1:      vld1.32         {q0},[r0,:128]
647         vld1.32         {q1},[r1,:128]
648         vsub.f32        q2,  q0,  q1
649         vadd.f32        q1,  q0,  q1
650         vst1.32         {q2},[r1,:128]!
651         vst1.32         {q1},[r0,:128]!
652         subs            r2,  r2,  #4
653         bgt             1b
654         bx              lr
655 endfunc
656
657 function ff_scalarproduct_float_neon, export=1
658         vmov.f32        q2,  #0.0
659 1:      vld1.32         {q0},[r0,:128]!
660         vld1.32         {q1},[r1,:128]!
661         vmla.f32        q2,  q0,  q1
662         subs            r2,  r2,  #4
663         bgt             1b
664         vadd.f32        d0,  d4,  d5
665         vpadd.f32       d0,  d0,  d0
666 NOVFP   vmov.32         r0,  d0[0]
667         bx              lr
668 endfunc
669
670 function ff_vector_fmul_reverse_neon, export=1
671         add             r2,  r2,  r3,  lsl #2
672         sub             r2,  r2,  #32
673         mov             r12, #-32
674         vld1.32         {q0-q1},  [r1,:128]!
675         vld1.32         {q2-q3},  [r2,:128], r12
676 1:      pld             [r1, #32]
677         vrev64.32       q3,  q3
678         vmul.f32        d16, d0,  d7
679         vmul.f32        d17, d1,  d6
680         pld             [r2, #-32]
681         vrev64.32       q2,  q2
682         vmul.f32        d18, d2,  d5
683         vmul.f32        d19, d3,  d4
684         subs            r3,  r3,  #8
685         beq             2f
686         vld1.32         {q0-q1},  [r1,:128]!
687         vld1.32         {q2-q3},  [r2,:128], r12
688         vst1.32         {q8-q9},  [r0,:128]!
689         b               1b
690 2:      vst1.32         {q8-q9},  [r0,:128]!
691         bx              lr
692 endfunc
693
694 function ff_vector_fmul_add_neon, export=1
695         ldr             r12, [sp]
696         vld1.32         {q0-q1},  [r1,:128]!
697         vld1.32         {q8-q9},  [r2,:128]!
698         vld1.32         {q2-q3},  [r3,:128]!
699         vmul.f32        q10, q0,  q8
700         vmul.f32        q11, q1,  q9
701 1:      vadd.f32        q12, q2,  q10
702         vadd.f32        q13, q3,  q11
703         pld             [r1, #16]
704         pld             [r2, #16]
705         pld             [r3, #16]
706         subs            r12, r12, #8
707         beq             2f
708         vld1.32         {q0},     [r1,:128]!
709         vld1.32         {q8},     [r2,:128]!
710         vmul.f32        q10, q0,  q8
711         vld1.32         {q1},     [r1,:128]!
712         vld1.32         {q9},     [r2,:128]!
713         vmul.f32        q11, q1,  q9
714         vld1.32         {q2-q3},  [r3,:128]!
715         vst1.32         {q12-q13},[r0,:128]!
716         b               1b
717 2:      vst1.32         {q12-q13},[r0,:128]!
718         bx              lr
719 endfunc
720
721 function ff_vector_clipf_neon, export=1
722 VFP     vdup.32         q1,  d0[1]
723 VFP     vdup.32         q0,  d0[0]
724 NOVFP   vdup.32         q0,  r2
725 NOVFP   vdup.32         q1,  r3
726 NOVFP   ldr             r2,  [sp]
727         vld1.f32        {q2},[r1,:128]!
728         vmin.f32        q10, q2,  q1
729         vld1.f32        {q3},[r1,:128]!
730         vmin.f32        q11, q3,  q1
731 1:      vmax.f32        q8,  q10, q0
732         vmax.f32        q9,  q11, q0
733         subs            r2,  r2,  #8
734         beq             2f
735         vld1.f32        {q2},[r1,:128]!
736         vmin.f32        q10, q2,  q1
737         vld1.f32        {q3},[r1,:128]!
738         vmin.f32        q11, q3,  q1
739         vst1.f32        {q8},[r0,:128]!
740         vst1.f32        {q9},[r0,:128]!
741         b               1b
742 2:      vst1.f32        {q8},[r0,:128]!
743         vst1.f32        {q9},[r0,:128]!
744         bx              lr
745 endfunc
746
747 function ff_apply_window_int16_neon, export=1
748         push            {r4,lr}
749         add             r4,  r1,  r3,  lsl #1
750         add             lr,  r0,  r3,  lsl #1
751         sub             r4,  r4,  #16
752         sub             lr,  lr,  #16
753         mov             r12, #-16
754 1:
755         vld1.16         {q0},     [r1,:128]!
756         vld1.16         {q2},     [r2,:128]!
757         vld1.16         {q1},     [r4,:128], r12
758         vrev64.16       q3,  q2
759         vqrdmulh.s16    q0,  q0,  q2
760         vqrdmulh.s16    d2,  d2,  d7
761         vqrdmulh.s16    d3,  d3,  d6
762         vst1.16         {q0},     [r0,:128]!
763         vst1.16         {q1},     [lr,:128], r12
764         subs            r3,  r3,  #16
765         bgt             1b
766
767         pop             {r4,pc}
768 endfunc
769
770 function ff_vector_clip_int32_neon, export=1
771         vdup.32         q0,  r2
772         vdup.32         q1,  r3
773         ldr             r2,  [sp]
774 1:
775         vld1.32         {q2-q3},  [r1,:128]!
776         vmin.s32        q2,  q2,  q1
777         vmin.s32        q3,  q3,  q1
778         vmax.s32        q2,  q2,  q0
779         vmax.s32        q3,  q3,  q0
780         vst1.32         {q2-q3},  [r0,:128]!
781         subs            r2,  r2,  #8
782         bgt             1b
783         bx              lr
784 endfunc