innoHeim/Rsupport Screen Capture Codec decoder
[ffmpeg.git] / libavcodec / aarch64 / imdct15_neon.S
1 /*
2  * Copyright (c) 2014 Janne Grunau <janne-libav@jannau.net>
3  *
4  * This file is part of Libav.
5  *
6  * Libav is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * Libav is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with Libav; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #include "libavutil/aarch64/asm.S"
22
23 #include "asm-offsets.h"
24
25 .macro shuffle a, b, c, d
26 const shuffle_\a\b\c\d, align=4
27         .byte (\a * 4), (\a * 4 + 1), (\a * 4 + 2), (\a * 4 + 3)
28         .byte (\b * 4), (\b * 4 + 1), (\b * 4 + 2), (\b * 4 + 3)
29         .byte (\c * 4), (\c * 4 + 1), (\c * 4 + 2), (\c * 4 + 3)
30         .byte (\d * 4), (\d * 4 + 1), (\d * 4 + 2), (\d * 4 + 3)
31 endconst
32 .endm
33
34 shuffle 0, 2, 1, 3
35 shuffle 1, 0, 3, 2
36 shuffle 2, 3, 0, 1
37 shuffle 3, 1, 2, 0
38
39
40 function fft5_neon
41         lsl             x2,  x2,  #3
42         ld1             {v24.2s},         [x1],  x2
43         ld2             {v25.s,v26.s}[0], [x1],  x2
44         ld2             {v25.s,v26.s}[1], [x1],  x2
45         ld2             {v25.s,v26.s}[2], [x1],  x2
46         ld2             {v25.s,v26.s}[3], [x1]
47         dup             v6.4s,  v24.s[0]
48         dup             v7.4s,  v24.s[1]
49
50         faddp           v0.4s,  v25.4s, v26.4s
51         // z[][0], z[][3]
52         fmul            v16.4s, v25.4s, v15.s[0] // rr
53         fmul            v17.4s, v25.4s, v15.s[1] // ri
54         fmul            v18.4s, v26.4s, v15.s[0] // ir
55         fmul            v19.4s, v26.4s, v15.s[1] // ii
56         faddp           v0.4s,  v0.4s,  v0.4s
57         // z[][1], z[][2]
58         fmul            v20.4s, v25.4s, v15.s[2] // rr
59         fmul            v21.4s, v25.4s, v15.s[3] // ri
60         fmul            v22.4s, v26.4s, v15.s[2] // ir
61         fmul            v23.4s, v26.4s, v15.s[3] // ii
62         fadd            v0.2s,  v24.2s, v0.2s   // out[0]
63
64         // z[0123][0], z[0123][3]
65         fsub            v24.4s, v16.4s, v19.4s  //    (c).re =  rr - ii;
66         fadd            v27.4s, v16.4s, v19.4s  //    (d).re =  rr + ii;
67         ld1             {v16.16b},  [x11]
68         ld1             {v19.16b},  [x14]
69         fadd            v28.4s, v17.4s, v18.4s  //    (c).im =  ri + ir;
70         fsub            v31.4s, v18.4s, v17.4s  //    (d).im = -ri + ir;
71         ld1             {v17.16b},  [x12]
72         // z[0123][1], z[0123][2]
73         fsub            v25.4s, v20.4s, v23.4s  //    (c).re =  rr - ii;
74         fadd            v26.4s, v20.4s, v23.4s  //    (d).re =  rr + ii;
75         ld1             {v18.16b},  [x13]
76         fadd            v29.4s, v21.4s, v22.4s  //    (c).im =  ri + ir;
77         fsub            v30.4s, v22.4s, v21.4s  //    (d).im = -ri + ir;
78
79         //real
80         tbl             v20.16b, {v24.16b}, v16.16b
81         tbl             v21.16b, {v25.16b}, v17.16b
82         tbl             v22.16b, {v26.16b}, v18.16b
83         tbl             v23.16b, {v27.16b}, v19.16b
84         //imag
85         tbl             v16.16b, {v28.16b}, v16.16b
86         tbl             v17.16b, {v29.16b}, v17.16b
87         tbl             v18.16b, {v30.16b}, v18.16b
88         tbl             v19.16b, {v31.16b}, v19.16b
89
90         fadd            v6.4s,  v6.4s,  v20.4s
91         fadd            v22.4s, v22.4s, v23.4s
92         fadd            v7.4s,  v7.4s,  v16.4s
93         fadd            v18.4s, v18.4s, v19.4s
94
95         fadd            v21.4s, v21.4s, v22.4s
96         fadd            v17.4s, v17.4s, v18.4s
97         fadd            v6.4s,  v6.4s,  v21.4s
98         fadd            v7.4s,  v7.4s,  v17.4s
99
100         ret
101 endfunc
102
103 function fft15_neon
104         mov             x8,  x1
105         mov             x9,  x30
106         add             x2,  x3,  x3,  lsl #1   // 3 * stride
107
108         add             x1,  x8,  x3,  lsl #3   // in + 1 * stride
109         bl              fft5_neon
110         mov             v1.8b,   v0.8b
111         mov             v2.16b,  v6.16b
112         mov             v3.16b,  v7.16b
113
114         add             x1,  x8,  x3,  lsl #4   // in + 2 * stride
115         add             x2,  x3,  x3,  lsl #1   // 3 * stride
116         bl              fft5_neon
117         zip1            v1.4s,   v1.4s,  v0.4s
118         mov             v4.16b,  v6.16b
119         mov             v5.16b,  v7.16b
120
121         mov             x1,  x8                 // in + 0 * stride
122         add             x2,  x3,  x3,  lsl #1   // 3 * stride
123         bl              fft5_neon
124
125         faddp           v20.4s, v1.4s,  v1.4s
126
127         ext             v18.16b, v8.16b,  v8.16b,  #4
128         ext             v19.16b, v9.16b,  v9.16b,  #4
129         mov             v16.16b, v6.16b
130         mov             v17.16b, v7.16b
131         fadd            v20.2s, v20.2s, v0.2s
132
133         uzp1            v18.4s, v18.4s, v10.4s  // exp[2,4,6,8].re
134         uzp1            v19.4s, v19.4s, v11.4s  // exp[2,4,6,8].im
135
136         st1             {v20.2s},  [x0], #8     // out[0]
137
138         fmla            v16.4s, v2.4s,  v8.4s
139         fmls            v16.4s, v3.4s,  v9.4s
140
141         fmla            v17.4s, v2.4s,  v9.4s
142         fmla            v17.4s, v3.4s,  v8.4s
143
144         fmla            v16.4s, v4.4s,  v18.4s
145         fmls            v16.4s, v5.4s,  v19.4s
146
147         fmla            v17.4s, v4.4s,  v19.4s
148         fmla            v17.4s, v5.4s,  v18.4s
149
150         zip1            v18.4s, v16.4s, v17.4s
151         zip2            v19.4s, v16.4s, v17.4s
152
153         rev64           v31.4s, v14.4s
154         trn1            v28.2d, v1.2d,  v1.2d
155         trn2            v29.2d, v1.2d,  v1.2d
156         zip1            v30.2d, v14.2d, v31.2d
157         zip2            v31.2d, v14.2d, v31.2d
158
159         st1             {v18.4s,v19.4s},  [x0], #32 // out[1-4]
160
161         fmul            v16.4s, v28.4s, v30.4s
162         fmul            v17.4s, v29.4s, v30.4s
163         fmls            v16.4s, v29.4s, v31.4s
164         fmla            v17.4s, v28.4s, v31.4s
165         faddp           v16.4s, v16.4s, v16.4s
166         faddp           v17.4s, v17.4s, v17.4s
167         zip1            v18.2s, v16.2s, v17.2s
168         zip2            v19.2s, v16.2s, v17.2s
169
170         fadd            v18.2s, v18.2s, v0.2s
171         fadd            v0.2s,  v19.2s, v0.2s
172
173         ext             v30.16b, v12.16b, v12.16b, #4
174         ext             v31.16b, v13.16b, v13.16b, #4
175         mov             v16.16b, v6.16b
176         mov             v17.16b, v7.16b
177
178         uzp1            v30.4s, v30.4s, v8.4s
179         uzp1            v31.4s, v31.4s, v9.4s
180
181         st1             {v18.2s},  [x0], #8     // out[5]
182
183         fmla            v16.4s, v2.4s,  v10.4s
184         fmls            v16.4s, v3.4s,  v11.4s
185
186         fmla            v17.4s, v2.4s,  v11.4s
187         fmla            v17.4s, v3.4s,  v10.4s
188
189         fmla            v16.4s, v4.4s,  v30.4s
190         fmls            v16.4s, v5.4s,  v31.4s
191
192         fmla            v17.4s, v4.4s,  v31.4s
193         fmla            v17.4s, v5.4s,  v30.4s
194
195         zip1            v18.4s, v16.4s, v17.4s
196         zip2            v19.4s, v16.4s, v17.4s
197
198         ext             v30.16b, v10.16b, v10.16b, #4
199         ext             v31.16b, v11.16b, v11.16b, #4
200
201         fmla            v6.4s,  v2.4s,  v12.4s
202         fmls            v6.4s,  v3.4s,  v13.4s
203
204         st1             {v18.4s,v19.4s},  [x0], #32 // out[6-9]
205
206         uzp1            v30.4s, v30.4s, v12.4s
207         uzp1            v31.4s, v31.4s, v13.4s
208
209         fmla            v7.4s,  v2.4s,  v13.4s
210         fmla            v7.4s,  v3.4s,  v12.4s
211
212         st1             {v0.2s},  [x0], #8     // out[10]
213
214         fmla            v6.4s,  v4.4s,  v30.4s
215         fmls            v6.4s,  v5.4s,  v31.4s
216
217         fmla            v7.4s,  v4.4s,  v31.4s
218         fmla            v7.4s,  v5.4s,  v30.4s
219
220         zip1            v18.4s, v6.4s,  v7.4s
221         zip2            v19.4s, v6.4s,  v7.4s
222
223         st1             {v18.4s,v19.4s},  [x0], #32 // out[11-14]
224
225         ret             x9
226 endfunc
227
228 // x0: out, x1: out+len2, x2: exptab, x3: len2
229 function fft15_pass
230         ands            x6,  x3,  #3
231         mov             x4,  x0
232         mov             x5,  x1
233         b.eq            9f
234         ld1             {v0.2s},  [x0], #8
235         ld1             {v1.2s},  [x1], #8
236         sub             x3,  x3,  x6
237         subs            x6,  x6,  #1
238         fadd            v2.2s,  v0.2s,  v1.2s
239         fsub            v3.2s,  v0.2s,  v1.2s
240         add             x2,  x2,  #8
241         st1             {v2.2s},  [x4], #8
242         st1             {v3.2s},  [x5], #8
243         b.eq            9f
244 1:
245         subs            x6,  x6,  #1
246         ldp             s4,  s5,  [x2], #8
247         ldp             s2,  s3,  [x1], #8
248         ldp             s0,  s1,  [x0], #8
249
250         fmul            s6,  s2,  s4
251         fmul            s7,  s2,  s5
252         fmls            s6,  s3,  v5.s[0]
253         fmla            s7,  s3,  v4.s[0]
254
255         fsub            s2,  s0,  s6
256         fsub            s3,  s1,  s7
257         fadd            s0,  s0,  s6
258         fadd            s1,  s1,  s7
259
260         stp             s2,  s3,  [x5], #8
261         stp             s0,  s1,  [x4], #8
262         b.gt            1b
263 9:
264         ld1             {v4.4s,v5.4s}, [x2],  #32
265         ld2             {v2.4s,v3.4s}, [x1],  #32
266         uzp1            v6.4s,  v4.4s,  v5.4s
267         uzp2            v7.4s,  v4.4s,  v5.4s
268         ld2             {v0.4s,v1.4s}, [x0],  #32
269 8:
270         subs            x3,  x3,  #8
271
272         fmul            v4.4s,  v2.4s,  v6.4s
273         fmul            v5.4s,  v2.4s,  v7.4s
274         b.lt            4f
275
276         ld1             {v18.4s,v19.4s}, [x2],  #32
277
278         fmls            v4.4s,  v3.4s,  v7.4s
279         fmla            v5.4s,  v3.4s,  v6.4s
280
281         ld2             {v22.4s,v23.4s}, [x1],  #32
282
283         fsub            v2.4s,  v0.4s,  v4.4s
284         fadd            v0.4s,  v0.4s,  v4.4s
285         fsub            v3.4s,  v1.4s,  v5.4s
286         fadd            v1.4s,  v1.4s,  v5.4s
287
288         uzp1            v16.4s, v18.4s, v19.4s
289         uzp2            v17.4s, v18.4s, v19.4s
290
291         st2             {v2.4s,v3.4s}, [x5],  #32
292         st2             {v0.4s,v1.4s}, [x4],  #32
293         ld2             {v20.4s,v21.4s}, [x0],  #32
294
295         fmul            v18.4s, v22.4s, v16.4s
296         fmul            v19.4s, v22.4s, v17.4s
297         b.eq            0f
298
299         ld1             {v4.4s,v5.4s}, [x2],  #32
300
301         fmls            v18.4s, v23.4s, v17.4s
302         fmla            v19.4s, v23.4s, v16.4s
303
304         ld2             {v2.4s,v3.4s}, [x1],  #32
305
306         fsub            v22.4s, v20.4s, v18.4s
307         fadd            v20.4s, v20.4s, v18.4s
308         fsub            v23.4s, v21.4s, v19.4s
309         fadd            v21.4s, v21.4s, v19.4s
310
311         uzp1            v6.4s,  v4.4s,  v5.4s
312         uzp2            v7.4s,  v4.4s,  v5.4s
313
314         st2             {v22.4s,v23.4s}, [x5],  #32
315         st2             {v20.4s,v21.4s}, [x4],  #32
316         ld2             {v0.4s,v1.4s}, [x0],  #32
317
318         b               8b
319 4:
320         fmls            v4.4s,  v3.4s,  v7.4s
321         fmla            v5.4s,  v3.4s,  v6.4s
322
323         fsub            v2.4s,  v0.4s,  v4.4s
324         fadd            v0.4s,  v0.4s,  v4.4s
325         fsub            v3.4s,  v1.4s,  v5.4s
326         fadd            v1.4s,  v1.4s,  v5.4s
327
328         st2             {v2.4s,v3.4s}, [x5],  #32
329         st2             {v0.4s,v1.4s}, [x4],  #32
330
331         ret
332 0:
333         fmls            v18.4s, v23.4s, v17.4s
334         fmla            v19.4s, v23.4s, v16.4s
335
336         fsub            v22.4s, v20.4s, v18.4s
337         fadd            v20.4s, v20.4s, v18.4s
338         fsub            v23.4s, v21.4s, v19.4s
339         fadd            v21.4s, v21.4s, v19.4s
340
341         st2             {v22.4s,v23.4s}, [x5],  #32
342         st2             {v20.4s,v21.4s}, [x4],  #32
343
344         ret
345 endfunc
346
347 function fft30_neon, align=6
348         sub             sp,  sp,  #0x20
349         stp             x20, x21, [sp]
350         stp             x22, x30, [sp, #0x10]
351         mov             x21, x1
352         mov             x22, x2
353         mov             x20, x4
354         mov             x0,  x21
355         mov             x1,  x22
356         lsl             x3,  x20, #1
357         bl              fft15_neon
358
359         add             x0,  x21, #15*8
360         add             x1,  x22, x20,  lsl #3
361         lsl             x3,  x20, #1
362         bl              fft15_neon
363
364         ldr             x2,  [x10, #(CELT_EXPTAB + 8)]  // s->exptab[1]
365         add             x0,  x21, #0
366         add             x1,  x21, #15*8
367         mov             x3,  #15
368         ldp             x20, x21, [sp]
369         ldp             x22, x30, [sp, #0x10]
370         add             sp,  sp,  #0x20
371         b               fft15_pass
372 endfunc
373
374 .macro  def_fft n, n2
375 function fft\n\()_neon, align=6
376         sub             sp,  sp,  #0x30
377         stp             x20, x21, [sp]
378         stp             x22, x30, [sp, #0x10]
379         stp             x23, x24, [sp, #0x20]
380         mov             x21, x1
381         mov             x22, x2
382         mov             x23, x3
383         mov             x20, x4
384         sub             x3,  x3,  #1
385         lsl             x4,  x4,  #1
386         bl              fft\n2\()_neon
387
388         add             x1,  x21, #(\n2 * 8)
389         add             x2,  x22, x20, lsl #3
390         sub             x3,  x23, #1
391         lsl             x4,  x20, #1
392         bl              fft\n2\()_neon
393
394         add             x5,  x10, #CELT_EXPTAB
395         mov             x0,  x21
396         ldr             x2,  [x5,  x23, lsl #3] // s->exptab[N]
397         add             x1,  x21, #(\n2 * 8)
398         mov             x3,  #\n2
399         ldp             x20, x21, [sp]
400         ldp             x22, x30, [sp, #0x10]
401         ldp             x23, x24, [sp, #0x20]
402         add             sp,  sp,  #0x30
403         b               fft15_pass
404 endfunc
405 .endm
406
407         def_fft    60,  30
408         def_fft   120,  60
409         def_fft   240, 120
410         def_fft   480, 240
411         def_fft   960, 480
412
413 function fft_b15_calc_neon
414         sub             sp,  sp,  #0x50
415         ldr             x8,  [x0,  #CELT_EXPTAB]    // s->exptab[0]
416         movrel          x6,  fact5
417         movrel          x11, shuffle_0213
418         movrel          x12, shuffle_1032
419         movrel          x13, shuffle_2301
420         movrel          x14, shuffle_3120
421         add             x8,  x8,  #8
422         movrel          x5,  fft_tab_neon
423         stp             x20, x30, [sp]
424         stp             d8,  d9,  [sp, #0x10]
425         stp             d10, d11, [sp, #0x20]
426         stp             d12, d13, [sp, #0x30]
427         stp             d14, d15, [sp, #0x40]
428         ld1             {v15.4s}, [x6]
429         ld1             {v0.4s,v1.4s},   [x8],  #32
430         ld1             {v6.2s},  [x8],  #8
431         ld1             {v2.4s,v3.4s},   [x8],  #32
432         ld1             {v7.2s},  [x8],  #8
433         ld1             {v4.4s,v5.4s},   [x8],  #32
434         uzp1            v8.4s,  v0.4s,  v1.4s   // exp[ 1 -  4].re
435         uzp2            v9.4s,  v0.4s,  v1.4s   // exp[ 1 -  4].im
436         uzp1            v10.4s, v2.4s,  v3.4s   // exp[ 6 -  9].re
437         uzp2            v11.4s, v2.4s,  v3.4s   // exp[ 6 -  9].im
438         uzp1            v12.4s, v4.4s,  v5.4s   // exp[11 - 14].re
439         uzp2            v13.4s, v4.4s,  v5.4s   // exp[11 - 14].im
440         zip1            v14.4s, v6.4s,  v7.4s   // exp[5,10].re/exp[5,10].im
441         add             x5,  x5,  x3,  lsl #3
442         ldr             x5,  [x5]
443         mov             x10, x0
444         blr             x5
445         ldp             x20, x30, [sp]
446         ldp             d8,  d9,  [sp, #0x10]
447         ldp             d10, d11, [sp, #0x20]
448         ldp             d12, d13, [sp, #0x30]
449         ldp             d14, d15, [sp, #0x40]
450         add             sp,  sp,  #0x50
451         ret
452 endfunc
453
454 const   fft_tab_neon, relocate=1
455         .quad fft15_neon
456         .quad fft30_neon
457         .quad fft60_neon
458         .quad fft120_neon
459         .quad fft240_neon
460         .quad fft480_neon
461         .quad fft960_neon
462 endconst
463
464 function ff_celt_imdct_half_neon, export=1
465         sub             sp,  sp,  #0x20
466         stp             x21, x30, [sp]
467         str             s0, [sp, #0x10]
468
469         ldp             w5,  w6,  [x0,  #CELT_LEN2] // CELT_LEN4
470         mov             x10, x0
471         mov             x21, x1
472         sub             w5,  w5,  #1
473         lsl             x7,  x3,  #3            //  2 * stride * sizeof(float)
474         sub             x8,  xzr, x3,  lsl #3   // -2 * stride * sizeof(float)
475         mul             x5,  x5,  x3
476         ldp             x9,  x10, [x0,  #CELT_TMP]  // CELT_TWIDDLE
477         ldr             w3,  [x0, #CELT_FFT_N]
478         add             x5,  x2,  x5,  lsl #2
479         mov             x11, x9
480
481         sub             w6,  w6,  #4
482         ld1             {v0.s}[0],  [x5], x8
483         ld1             {v1.s}[0],  [x2], x7
484         ld1             {v4.4s,v5.4s}, [x10], #32
485         ld1             {v0.s}[1],  [x5], x8
486         ld1             {v1.s}[1],  [x2], x7
487         uzp1            v2.4s,  v4.4s,  v5.4s
488         ld1             {v0.s}[2],  [x5], x8
489         ld1             {v1.s}[2],  [x2], x7
490         uzp2            v3.4s,  v4.4s,  v5.4s
491         ld1             {v0.s}[3],  [x5], x8
492         ld1             {v1.s}[3],  [x2], x7
493 1:
494         subs            w6,  w6,  #4
495
496         ld1             {v20.s}[0], [x5], x8
497         ld1             {v21.s}[0], [x2], x7
498         ld1             {v4.4s,v5.4s}, [x10], #32
499
500         fmul            v6.4s,  v0.4s,  v2.4s
501         fmul            v7.4s,  v0.4s,  v3.4s
502
503         ld1             {v20.s}[1], [x5], x8
504         ld1             {v21.s}[1], [x2], x7
505
506         fmls            v6.4s,  v1.4s,  v3.4s
507         fmla            v7.4s,  v1.4s,  v2.4s
508
509         ld1             {v20.s}[2], [x5], x8
510         ld1             {v21.s}[2], [x2], x7
511
512         uzp1            v2.4s,  v4.4s,  v5.4s
513         uzp2            v3.4s,  v4.4s,  v5.4s
514         ld1             {v20.s}[3], [x5], x8
515         ld1             {v21.s}[3], [x2], x7
516
517         zip1            v4.4s,  v6.4s,  v7.4s
518         zip2            v5.4s,  v6.4s,  v7.4s
519
520         fmul            v6.4s,  v20.4s, v2.4s
521         fmul            v7.4s,  v20.4s, v3.4s
522
523         st1             {v4.4s,v5.4s}, [x9], #32
524
525         fmls            v6.4s,  v21.4s, v3.4s
526         fmla            v7.4s,  v21.4s, v2.4s
527
528         b.eq            3f
529
530         subs            w6,  w6,  #4
531         ld1             {v4.4s,v5.4s}, [x10], #32
532         ld1             {v0.s}[0],  [x5], x8
533         ld1             {v1.s}[0],  [x2], x7
534         uzp1            v2.4s,  v4.4s,  v5.4s
535         ld1             {v0.s}[1],  [x5], x8
536         ld1             {v1.s}[1],  [x2], x7
537         uzp2            v3.4s,  v4.4s,  v5.4s
538         ld1             {v0.s}[2],  [x5], x8
539         ld1             {v1.s}[2],  [x2], x7
540         zip1            v4.4s,  v6.4s,  v7.4s
541         zip2            v5.4s,  v6.4s,  v7.4s
542         ld1             {v0.s}[3],  [x5], x8
543         ld1             {v1.s}[3],  [x2], x7
544
545         st1             {v4.4s,v5.4s}, [x9], #32
546
547         b.gt            1b
548
549         fmul            v6.4s,  v0.4s,  v2.4s
550         fmul            v7.4s,  v0.4s,  v3.4s
551         fmls            v6.4s,  v1.4s,  v3.4s
552         fmla            v7.4s,  v1.4s,  v2.4s
553 3:
554         zip1            v4.4s,  v6.4s,  v7.4s
555         zip2            v5.4s,  v6.4s,  v7.4s
556         st1             {v4.4s,v5.4s}, [x9], #32
557
558         mov             x2,  x11
559         mov             x4,  #1
560
561         bl              fft_b15_calc_neon
562
563         ldr             w5,  [x10, #CELT_LEN4]
564         ldr             x6,  [x10, #CELT_TWIDDLE]
565         ldr             s31, [sp, #0x10]
566
567         add             x1,  x21, x5,  lsl #2
568         add             x3,  x6,  x5,  lsl #2
569         sub             x0,  x1,  #16
570         sub             x2,  x3,  #16
571         mov             x8,  #-16
572         mov             x7,  #16
573         mov             x10, x0
574         mov             x11, x1
575
576         sub             w5,  w5,  #4
577
578         ld1             {v0.4s},  [x0], x8
579         ld1             {v1.4s},  [x1], x7
580         ld1             {v2.4s},  [x2], x8
581         ld1             {v3.4s},  [x3], x7
582
583         uzp1            v4.4s,  v0.4s,  v1.4s   // z[-i-2, -i-1, +i, i+1].re
584         uzp2            v6.4s,  v0.4s,  v1.4s   // z[-i-2, -i-1, +i, i+1].im
585
586         uzp1            v5.4s,  v2.4s,  v3.4s   // twidlle_exptab[-i-2, -i-1, +i, i+1].re
587         uzp2            v7.4s,  v2.4s,  v3.4s   // twidlle_exptab[-i-2, -i-1, +i, i+1].im
588
589         fmul            v1.4s,  v6.4s,  v5.4s
590         fmul            v0.4s,  v6.4s,  v7.4s
591 2:
592         subs            w5,  w5,  #4
593
594         ld1             {v20.4s}, [x0], x8
595
596         fmla            v1.4s,  v4.4s,  v7.4s
597         fmls            v0.4s,  v4.4s,  v5.4s
598
599         ld1             {v21.4s}, [x1], x7
600
601         ext             v1.16b, v1.16b, v1.16b, #8
602         fmul            v0.4s,  v0.4s,  v31.s[0]
603
604         ld1             {v2.4s},  [x2], x8
605
606         rev64           v1.4s,  v1.4s
607         fmul            v1.4s,  v1.4s,  v31.s[0]
608
609         ld1             {v3.4s},  [x3], x7
610
611         zip1            v5.4s,  v0.4s,  v1.4s
612         zip2            v7.4s,  v0.4s,  v1.4s
613
614         uzp1            v4.4s,  v20.4s, v21.4s  // z[-i-2, -i-1, +i, i+1].re
615         uzp2            v6.4s,  v20.4s, v21.4s  // z[-i-2, -i-1, +i, i+1].im
616
617         st1             {v5.4s},  [x10], x8
618         st1             {v7.4s},  [x11], x7
619
620         uzp1            v5.4s,  v2.4s,  v3.4s   // twidlle_exptab[-i-2, -i-1, +i, i+1].re
621         uzp2            v7.4s,  v2.4s,  v3.4s   // twidlle_exptab[-i-2, -i-1, +i, i+1].im
622
623         fmul            v1.4s,  v6.4s,  v5.4s
624         fmul            v0.4s,  v6.4s,  v7.4s
625         b.gt            2b
626
627         fmla            v1.4s,  v4.4s,  v7.4s
628         fmls            v0.4s,  v4.4s,  v5.4s
629         ext             v1.16b, v1.16b, v1.16b, #8
630         fmul            v0.4s,  v0.4s,  v31.s[0]
631         rev64           v1.4s,  v1.4s
632         fmul            v1.4s,  v1.4s,  v31.s[0]
633         zip1            v5.4s,  v0.4s,  v1.4s
634         zip2            v7.4s,  v0.4s,  v1.4s
635         st1             {v5.4s},  [x10], x8
636         st1             {v7.4s},  [x11], x7
637
638         ldp             x21, x30, [sp]
639         add             sp,  sp,  #0x20
640         ret
641 endfunc
642
643 // [0] = exp(2 * i * pi / 5), [1] = exp(2 * i * pi * 2 / 5)
644 const   fact5,          align=4
645         .float           0.30901699437494745, 0.95105651629515353
646         .float          -0.80901699437494734, 0.58778525229247325
647 endconst