Merge commit '42cf7f91f1e9dabf494ff469d8f67ac8b33b0f63'
[ffmpeg.git] / libavcodec / arm / hpeldsp_neon.S
1 /*
2  * ARM NEON optimised DSP functions
3  * Copyright (c) 2008 Mans Rullgard <mans@mansr.com>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 #include "libavutil/arm/asm.S"
23
24 .macro  pixels16        rnd=1, avg=0
25   .if \avg
26         mov             r12, r0
27   .endif
28 1:      vld1.8          {q0},     [r1], r2
29         vld1.8          {q1},     [r1], r2
30         vld1.8          {q2},     [r1], r2
31         pld             [r1, r2, lsl #2]
32         vld1.8          {q3},     [r1], r2
33         pld             [r1]
34         pld             [r1, r2]
35         pld             [r1, r2, lsl #1]
36   .if \avg
37         vld1.8          {q8},     [r12,:128], r2
38         vrhadd.u8       q0,  q0,  q8
39         vld1.8          {q9},     [r12,:128], r2
40         vrhadd.u8       q1,  q1,  q9
41         vld1.8          {q10},    [r12,:128], r2
42         vrhadd.u8       q2,  q2,  q10
43         vld1.8          {q11},    [r12,:128], r2
44         vrhadd.u8       q3,  q3,  q11
45   .endif
46         subs            r3,  r3,  #4
47         vst1.64         {q0},     [r0,:128], r2
48         vst1.64         {q1},     [r0,:128], r2
49         vst1.64         {q2},     [r0,:128], r2
50         vst1.64         {q3},     [r0,:128], r2
51         bne             1b
52         bx              lr
53 .endm
54
55 .macro  pixels16_x2     rnd=1, avg=0
56 1:      vld1.8          {d0-d2},  [r1], r2
57         vld1.8          {d4-d6},  [r1], r2
58         pld             [r1]
59         pld             [r1, r2]
60         subs            r3,  r3,  #2
61         vext.8          q1,  q0,  q1,  #1
62         avg             q0,  q0,  q1
63         vext.8          q3,  q2,  q3,  #1
64         avg             q2,  q2,  q3
65   .if \avg
66         vld1.8          {q1},     [r0,:128], r2
67         vld1.8          {q3},     [r0,:128]
68         vrhadd.u8       q0,  q0,  q1
69         vrhadd.u8       q2,  q2,  q3
70         sub             r0,  r0,  r2
71   .endif
72         vst1.8          {q0},     [r0,:128], r2
73         vst1.8          {q2},     [r0,:128], r2
74         bne             1b
75         bx              lr
76 .endm
77
78 .macro  pixels16_y2     rnd=1, avg=0
79         sub             r3,  r3,  #2
80         vld1.8          {q0},     [r1], r2
81         vld1.8          {q1},     [r1], r2
82 1:      subs            r3,  r3,  #2
83         avg             q2,  q0,  q1
84         vld1.8          {q0},     [r1], r2
85         avg             q3,  q0,  q1
86         vld1.8          {q1},     [r1], r2
87         pld             [r1]
88         pld             [r1, r2]
89   .if \avg
90         vld1.8          {q8},     [r0,:128], r2
91         vld1.8          {q9},     [r0,:128]
92         vrhadd.u8       q2,  q2,  q8
93         vrhadd.u8       q3,  q3,  q9
94         sub             r0,  r0,  r2
95   .endif
96         vst1.8          {q2},     [r0,:128], r2
97         vst1.8          {q3},     [r0,:128], r2
98         bne             1b
99
100         avg             q2,  q0,  q1
101         vld1.8          {q0},     [r1], r2
102         avg             q3,  q0,  q1
103   .if \avg
104         vld1.8          {q8},     [r0,:128], r2
105         vld1.8          {q9},     [r0,:128]
106         vrhadd.u8       q2,  q2,  q8
107         vrhadd.u8       q3,  q3,  q9
108         sub             r0,  r0,  r2
109   .endif
110         vst1.8          {q2},     [r0,:128], r2
111         vst1.8          {q3},     [r0,:128], r2
112
113         bx              lr
114 .endm
115
116 .macro  pixels16_xy2    rnd=1, avg=0
117         sub             r3,  r3,  #2
118         vld1.8          {d0-d2},  [r1], r2
119         vld1.8          {d4-d6},  [r1], r2
120 NRND    vmov.i16        q13, #1
121         pld             [r1]
122         pld             [r1, r2]
123         vext.8          q1,  q0,  q1,  #1
124         vext.8          q3,  q2,  q3,  #1
125         vaddl.u8        q8,  d0,  d2
126         vaddl.u8        q10, d1,  d3
127         vaddl.u8        q9,  d4,  d6
128         vaddl.u8        q11, d5,  d7
129 1:      subs            r3,  r3,  #2
130         vld1.8          {d0-d2},  [r1], r2
131         vadd.u16        q12, q8,  q9
132         pld             [r1]
133 NRND    vadd.u16        q12, q12, q13
134         vext.8          q15, q0,  q1,  #1
135         vadd.u16        q1 , q10, q11
136         shrn            d28, q12, #2
137 NRND    vadd.u16        q1,  q1,  q13
138         shrn            d29, q1,  #2
139   .if \avg
140         vld1.8          {q8},     [r0,:128]
141         vrhadd.u8       q14, q14, q8
142   .endif
143         vaddl.u8        q8,  d0,  d30
144         vld1.8          {d2-d4},  [r1], r2
145         vaddl.u8        q10, d1,  d31
146         vst1.8          {q14},    [r0,:128], r2
147         vadd.u16        q12, q8,  q9
148         pld             [r1, r2]
149 NRND    vadd.u16        q12, q12, q13
150         vext.8          q2,  q1,  q2,  #1
151         vadd.u16        q0,  q10, q11
152         shrn            d30, q12, #2
153 NRND    vadd.u16        q0,  q0,  q13
154         shrn            d31, q0,  #2
155   .if \avg
156         vld1.8          {q9},     [r0,:128]
157         vrhadd.u8       q15, q15, q9
158   .endif
159         vaddl.u8        q9,  d2,  d4
160         vaddl.u8        q11, d3,  d5
161         vst1.8          {q15},    [r0,:128], r2
162         bgt             1b
163
164         vld1.8          {d0-d2},  [r1], r2
165         vadd.u16        q12, q8,  q9
166 NRND    vadd.u16        q12, q12, q13
167         vext.8          q15, q0,  q1,  #1
168         vadd.u16        q1 , q10, q11
169         shrn            d28, q12, #2
170 NRND    vadd.u16        q1,  q1,  q13
171         shrn            d29, q1,  #2
172   .if \avg
173         vld1.8          {q8},     [r0,:128]
174         vrhadd.u8       q14, q14, q8
175   .endif
176         vaddl.u8        q8,  d0,  d30
177         vaddl.u8        q10, d1,  d31
178         vst1.8          {q14},    [r0,:128], r2
179         vadd.u16        q12, q8,  q9
180 NRND    vadd.u16        q12, q12, q13
181         vadd.u16        q0,  q10, q11
182         shrn            d30, q12, #2
183 NRND    vadd.u16        q0,  q0,  q13
184         shrn            d31, q0,  #2
185   .if \avg
186         vld1.8          {q9},     [r0,:128]
187         vrhadd.u8       q15, q15, q9
188   .endif
189         vst1.8          {q15},    [r0,:128], r2
190
191         bx              lr
192 .endm
193
194 .macro  pixels8         rnd=1, avg=0
195 1:      vld1.8          {d0},     [r1], r2
196         vld1.8          {d1},     [r1], r2
197         vld1.8          {d2},     [r1], r2
198         pld             [r1, r2, lsl #2]
199         vld1.8          {d3},     [r1], r2
200         pld             [r1]
201         pld             [r1, r2]
202         pld             [r1, r2, lsl #1]
203   .if \avg
204         vld1.8          {d4},     [r0,:64], r2
205         vrhadd.u8       d0,  d0,  d4
206         vld1.8          {d5},     [r0,:64], r2
207         vrhadd.u8       d1,  d1,  d5
208         vld1.8          {d6},     [r0,:64], r2
209         vrhadd.u8       d2,  d2,  d6
210         vld1.8          {d7},     [r0,:64], r2
211         vrhadd.u8       d3,  d3,  d7
212         sub             r0,  r0,  r2,  lsl #2
213   .endif
214         subs            r3,  r3,  #4
215         vst1.8          {d0},     [r0,:64], r2
216         vst1.8          {d1},     [r0,:64], r2
217         vst1.8          {d2},     [r0,:64], r2
218         vst1.8          {d3},     [r0,:64], r2
219         bne             1b
220         bx              lr
221 .endm
222
223 .macro  pixels8_x2      rnd=1, avg=0
224 1:      vld1.8          {q0},     [r1], r2
225         vext.8          d1,  d0,  d1,  #1
226         vld1.8          {q1},     [r1], r2
227         vext.8          d3,  d2,  d3,  #1
228         pld             [r1]
229         pld             [r1, r2]
230         subs            r3,  r3,  #2
231         vswp            d1,  d2
232         avg             q0,  q0,  q1
233   .if \avg
234         vld1.8          {d4},     [r0,:64], r2
235         vld1.8          {d5},     [r0,:64]
236         vrhadd.u8       q0,  q0,  q2
237         sub             r0,  r0,  r2
238   .endif
239         vst1.8          {d0},     [r0,:64], r2
240         vst1.8          {d1},     [r0,:64], r2
241         bne             1b
242         bx              lr
243 .endm
244
245 .macro  pixels8_y2      rnd=1, avg=0
246         sub             r3,  r3,  #2
247         vld1.8          {d0},     [r1], r2
248         vld1.8          {d1},     [r1], r2
249 1:      subs            r3,  r3,  #2
250         avg             d4,  d0,  d1
251         vld1.8          {d0},     [r1], r2
252         avg             d5,  d0,  d1
253         vld1.8          {d1},     [r1], r2
254         pld             [r1]
255         pld             [r1, r2]
256   .if \avg
257         vld1.8          {d2},     [r0,:64], r2
258         vld1.8          {d3},     [r0,:64]
259         vrhadd.u8       q2,  q2,  q1
260         sub             r0,  r0,  r2
261   .endif
262         vst1.8          {d4},     [r0,:64], r2
263         vst1.8          {d5},     [r0,:64], r2
264         bne             1b
265
266         avg             d4,  d0,  d1
267         vld1.8          {d0},     [r1], r2
268         avg             d5,  d0,  d1
269   .if \avg
270         vld1.8          {d2},     [r0,:64], r2
271         vld1.8          {d3},     [r0,:64]
272         vrhadd.u8       q2,  q2,  q1
273         sub             r0,  r0,  r2
274   .endif
275         vst1.8          {d4},     [r0,:64], r2
276         vst1.8          {d5},     [r0,:64], r2
277
278         bx              lr
279 .endm
280
281 .macro  pixels8_xy2     rnd=1, avg=0
282         sub             r3,  r3,  #2
283         vld1.8          {q0},     [r1], r2
284         vld1.8          {q1},     [r1], r2
285 NRND    vmov.i16        q11, #1
286         pld             [r1]
287         pld             [r1, r2]
288         vext.8          d4,  d0,  d1,  #1
289         vext.8          d6,  d2,  d3,  #1
290         vaddl.u8        q8,  d0,  d4
291         vaddl.u8        q9,  d2,  d6
292 1:      subs            r3,  r3,  #2
293         vld1.8          {q0},     [r1], r2
294         pld             [r1]
295         vadd.u16        q10, q8,  q9
296         vext.8          d4,  d0,  d1,  #1
297 NRND    vadd.u16        q10, q10, q11
298         vaddl.u8        q8,  d0,  d4
299         shrn            d5,  q10, #2
300         vld1.8          {q1},     [r1], r2
301         vadd.u16        q10, q8,  q9
302         pld             [r1, r2]
303   .if \avg
304         vld1.8          {d7},     [r0,:64]
305         vrhadd.u8       d5,  d5,  d7
306   .endif
307 NRND    vadd.u16        q10, q10, q11
308         vst1.8          {d5},     [r0,:64], r2
309         shrn            d7,  q10, #2
310   .if \avg
311         vld1.8          {d5},     [r0,:64]
312         vrhadd.u8       d7,  d7,  d5
313   .endif
314         vext.8          d6,  d2,  d3,  #1
315         vaddl.u8        q9,  d2,  d6
316         vst1.8          {d7},     [r0,:64], r2
317         bgt             1b
318
319         vld1.8          {q0},     [r1], r2
320         vadd.u16        q10, q8,  q9
321         vext.8          d4,  d0,  d1,  #1
322 NRND    vadd.u16        q10, q10, q11
323         vaddl.u8        q8,  d0,  d4
324         shrn            d5,  q10, #2
325         vadd.u16        q10, q8,  q9
326   .if \avg
327         vld1.8          {d7},     [r0,:64]
328         vrhadd.u8       d5,  d5,  d7
329   .endif
330 NRND    vadd.u16        q10, q10, q11
331         vst1.8          {d5},     [r0,:64], r2
332         shrn            d7,  q10, #2
333   .if \avg
334         vld1.8          {d5},     [r0,:64]
335         vrhadd.u8       d7,  d7,  d5
336   .endif
337         vst1.8          {d7},     [r0,:64], r2
338
339         bx              lr
340 .endm
341
342 .macro  pixfunc         pfx, name, suf, rnd=1, avg=0
343   .if \rnd
344     .macro avg  rd, rn, rm
345         vrhadd.u8       \rd, \rn, \rm
346     .endm
347     .macro shrn rd, rn, rm
348         vrshrn.u16      \rd, \rn, \rm
349     .endm
350     .macro NRND insn:vararg
351     .endm
352   .else
353     .macro avg  rd, rn, rm
354         vhadd.u8        \rd, \rn, \rm
355     .endm
356     .macro shrn rd, rn, rm
357         vshrn.u16       \rd, \rn, \rm
358     .endm
359     .macro NRND insn:vararg
360         \insn
361     .endm
362   .endif
363 function ff_\pfx\name\suf\()_neon, export=1
364         \name           \rnd, \avg
365 endfunc
366         .purgem         avg
367         .purgem         shrn
368         .purgem         NRND
369 .endm
370
371 .macro  pixfunc2        pfx, name, avg=0
372         pixfunc         \pfx, \name,          rnd=1, avg=\avg
373         pixfunc         \pfx, \name, _no_rnd, rnd=0, avg=\avg
374 .endm
375
376 function ff_put_h264_qpel16_mc00_neon, export=1
377         mov             r3,  #16
378 endfunc
379
380         pixfunc         put_, pixels16,     avg=0
381         pixfunc2        put_, pixels16_x2,  avg=0
382         pixfunc2        put_, pixels16_y2,  avg=0
383         pixfunc2        put_, pixels16_xy2, avg=0
384
385 function ff_avg_h264_qpel16_mc00_neon, export=1
386         mov             r3,  #16
387 endfunc
388
389         pixfunc         avg_, pixels16,     avg=1
390         pixfunc2        avg_, pixels16_x2,  avg=1
391         pixfunc2        avg_, pixels16_y2,  avg=1
392         pixfunc2        avg_, pixels16_xy2, avg=1
393
394 function ff_put_h264_qpel8_mc00_neon, export=1
395         mov             r3,  #8
396 endfunc
397
398         pixfunc         put_, pixels8,     avg=0
399         pixfunc2        put_, pixels8_x2,  avg=0
400         pixfunc2        put_, pixels8_y2,  avg=0
401         pixfunc2        put_, pixels8_xy2, avg=0
402
403 function ff_avg_h264_qpel8_mc00_neon, export=1
404         mov             r3,  #8
405 endfunc
406
407         pixfunc         avg_, pixels8,     avg=1
408         pixfunc         avg_, pixels8_x2,  avg=1
409         pixfunc         avg_, pixels8_y2,  avg=1
410         pixfunc         avg_, pixels8_xy2, avg=1