lavfi: port mp=fspp to a native libavfilter filter
[ffmpeg.git] / libavfilter / x86 / vf_fspp.c
1 /*
2  * Copyright (c) 2003 Michael Niedermayer <michaelni@gmx.at>
3  * Copyright (C) 2005 Nikolaj Poroshin <porosh3@psu.ru>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License along
18  * with FFmpeg; if not, write to the Free Software Foundation, Inc.,
19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
20  */
21
22 #include "libavutil/attributes.h"
23 #include "libavutil/cpu.h"
24 #include "libavutil/mem.h"
25 #include "libavutil/x86/asm.h"
26 #include "libavfilter/vf_fspp.h"
27
28 DECLARE_ALIGNED(32, static const uint8_t, dither)[8][8] = {
29     {  0,  48,  12,  60,   3,  51,  15,  63, },
30     { 32,  16,  44,  28,  35,  19,  47,  31, },
31     {  8,  56,   4,  52,  11,  59,   7,  55, },
32     { 40,  24,  36,  20,  43,  27,  39,  23, },
33     {  2,  50,  14,  62,   1,  49,  13,  61, },
34     { 34,  18,  46,  30,  33,  17,  45,  29, },
35     { 10,  58,   6,  54,   9,  57,   5,  53, },
36     { 42,  26,  38,  22,  41,  25,  37,  21, },
37 };
38
39 //This func reads from 1 slice, 1 and clears 0 & 1
40 static void store_slice_mmx(uint8_t *dst, int16_t *src,
41                             ptrdiff_t dst_stride, ptrdiff_t src_stride,
42                             ptrdiff_t width, ptrdiff_t height, ptrdiff_t log2_scale)
43 {
44     const uint8_t *od = &dither[0][0];
45     const uint8_t *end = &dither[height][0];
46     width = (width + 7) & ~7;
47     dst_stride -= width;
48
49     __asm__ volatile(
50         "mov %5 , %%"REG_d"                \n\t"
51         "mov %6 , %%"REG_S"                \n\t"
52         "mov %7 , %%"REG_D"                \n\t"
53         "mov %1 , %%"REG_a"                \n\t"
54         "movd %%"REG_d" , %%mm5            \n\t"
55         "xor $-1 , %%"REG_d"               \n\t"
56         "mov %%"REG_a" , %%"REG_c"         \n\t"
57         "add $7 , %%"REG_d"                \n\t"
58         "neg %%"REG_a"                     \n\t"
59         "sub %0 , %%"REG_c"                \n\t"
60         "add %%"REG_c" , %%"REG_c"         \n\t"
61         "movd %%"REG_d" , %%mm2            \n\t"
62         "mov %%"REG_c" , %1                \n\t"
63         "mov %2 , %%"REG_d"                \n\t"
64         "shl $4 , %%"REG_a"                \n\t"
65
66         "2:                                \n\t"
67         "movq (%%"REG_d") , %%mm3          \n\t"
68         "movq %%mm3 , %%mm4                \n\t"
69         "pxor %%mm7 , %%mm7                \n\t"
70         "punpcklbw %%mm7 , %%mm3           \n\t"
71         "punpckhbw %%mm7 , %%mm4           \n\t"
72         "mov %0 , %%"REG_c"                \n\t"
73         "psraw %%mm5 , %%mm3               \n\t"
74         "psraw %%mm5 , %%mm4               \n\t"
75         "1:                                \n\t"
76         "movq %%mm7, (%%"REG_S",%%"REG_a") \n\t"
77         "movq (%%"REG_S") , %%mm0          \n\t"
78         "movq 8(%%"REG_S"), %%mm1          \n\t"
79
80         "movq %%mm7, 8(%%"REG_S",%%"REG_a")\n\t"
81         "paddw %%mm3, %%mm0                \n\t"
82         "paddw %%mm4, %%mm1                \n\t"
83
84         "movq %%mm7, (%%"REG_S")           \n\t"
85         "psraw %%mm2, %%mm0                \n\t"
86         "psraw %%mm2, %%mm1                \n\t"
87
88         "movq %%mm7, 8(%%"REG_S")          \n\t"
89         "packuswb %%mm1, %%mm0             \n\t"
90         "add $16, %%"REG_S"                \n\t"
91
92         "movq %%mm0, (%%"REG_D")           \n\t"
93         "add $8, %%"REG_D"                 \n\t"
94         "sub $8, %%"REG_c"                 \n\t"
95         "jg 1b                             \n\t"
96         "add %1, %%"REG_S"                 \n\t"
97         "add $8, %%"REG_d"                 \n\t"
98         "add %3, %%"REG_D"                 \n\t"
99         "cmp %4, %%"REG_d"                 \n\t"
100         "jl 2b                             \n\t"
101
102         :
103         : "m" (width),      "m" (src_stride), "erm" (od), "m" (dst_stride), "erm" (end),
104           "m" (log2_scale), "m" (src),        "m" (dst)                                     //input
105         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
106         );
107 }
108
109 //This func reads from 2 slices, 0 & 2  and clears 2-nd
110 static void store_slice2_mmx(uint8_t *dst, int16_t *src,
111                              ptrdiff_t dst_stride, ptrdiff_t src_stride,
112                              ptrdiff_t width, ptrdiff_t height, ptrdiff_t log2_scale)
113 {
114     const uint8_t *od = &dither[0][0];
115     const uint8_t *end = &dither[height][0];
116     width = (width + 7) & ~7;
117     dst_stride -= width;
118
119     __asm__ volatile(
120         "mov %5, %%"REG_d"                \n\t"
121         "mov %6, %%"REG_S"                \n\t"
122         "mov %7, %%"REG_D"                \n\t"
123         "mov %1, %%"REG_a"                \n\t"
124         "movd %%"REG_d", %%mm5            \n\t"
125         "xor $-1, %%"REG_d"               \n\t"
126         "mov %%"REG_a", %%"REG_c"         \n\t"
127         "add $7, %%"REG_d"                \n\t"
128         "sub %0, %%"REG_c"                \n\t"
129         "add %%"REG_c", %%"REG_c"         \n\t"
130         "movd %%"REG_d", %%mm2            \n\t"
131         "mov %%"REG_c", %1                \n\t"
132         "mov %2, %%"REG_d"                \n\t"
133         "shl $5, %%"REG_a"                \n\t"
134
135         "2:                               \n\t"
136         "movq (%%"REG_d"), %%mm3          \n\t"
137         "movq %%mm3, %%mm4                \n\t"
138         "pxor %%mm7, %%mm7                \n\t"
139         "punpcklbw %%mm7, %%mm3           \n\t"
140         "punpckhbw %%mm7, %%mm4           \n\t"
141         "mov %0, %%"REG_c"                \n\t"
142         "psraw %%mm5, %%mm3               \n\t"
143         "psraw %%mm5, %%mm4               \n\t"
144         "1:                               \n\t"
145         "movq (%%"REG_S"), %%mm0          \n\t"
146         "movq 8(%%"REG_S"), %%mm1         \n\t"
147         "paddw %%mm3, %%mm0               \n\t"
148
149         "paddw (%%"REG_S",%%"REG_a"),%%mm0\n\t"
150         "paddw %%mm4, %%mm1               \n\t"
151         "movq 8(%%"REG_S",%%"REG_a"),%%mm6\n\t"
152
153         "movq %%mm7, (%%"REG_S",%%"REG_a")\n\t"
154         "psraw %%mm2, %%mm0               \n\t"
155         "paddw %%mm6, %%mm1               \n\t"
156
157         "movq %%mm7,8(%%"REG_S",%%"REG_a")\n\t"
158         "psraw %%mm2, %%mm1               \n\t"
159         "packuswb %%mm1, %%mm0            \n\t"
160
161         "movq %%mm0, (%%"REG_D")          \n\t"
162         "add $16, %%"REG_S"               \n\t"
163         "add $8, %%"REG_D"                \n\t"
164         "sub $8, %%"REG_c"                \n\t"
165         "jg 1b                            \n\t"
166         "add %1, %%"REG_S"                \n\t"
167         "add $8, %%"REG_d"                \n\t"
168         "add %3, %%"REG_D"                \n\t"
169         "cmp %4, %%"REG_d"                \n\t"
170         "jl 2b                            \n\t"
171
172         :
173         : "m" (width),      "m" (src_stride), "erm" (od), "m" (dst_stride), "erm" (end),
174           "m" (log2_scale), "m" (src),        "m" (dst)                                     //input
175         : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_D, "%"REG_S
176         );
177 }
178
179 static void mul_thrmat_mmx(FSPPContext *p, int q)
180 {
181     uint64_t *adr = &p->threshold_mtx_noq[0];
182
183     __asm__ volatile(
184         "movd %0, %%mm7                   \n\t"
185         "add $8*8*2, %%"REG_D"            \n\t"
186         "movq 0*8(%%"REG_S"), %%mm0       \n\t"
187         "punpcklwd %%mm7, %%mm7           \n\t"
188         "movq 1*8(%%"REG_S"), %%mm1       \n\t"
189         "punpckldq %%mm7, %%mm7           \n\t"
190         "pmullw %%mm7, %%mm0              \n\t"
191
192         "movq 2*8(%%"REG_S"), %%mm2       \n\t"
193         "pmullw %%mm7, %%mm1              \n\t"
194
195         "movq 3*8(%%"REG_S"), %%mm3       \n\t"
196         "pmullw %%mm7, %%mm2              \n\t"
197
198         "movq %%mm0, 0*8(%%"REG_D")       \n\t"
199         "movq 4*8(%%"REG_S"), %%mm4       \n\t"
200         "pmullw %%mm7, %%mm3              \n\t"
201
202         "movq %%mm1, 1*8(%%"REG_D")       \n\t"
203         "movq 5*8(%%"REG_S"), %%mm5       \n\t"
204         "pmullw %%mm7, %%mm4              \n\t"
205
206         "movq %%mm2, 2*8(%%"REG_D")       \n\t"
207         "movq 6*8(%%"REG_S"), %%mm6       \n\t"
208         "pmullw %%mm7, %%mm5              \n\t"
209
210         "movq %%mm3, 3*8(%%"REG_D")       \n\t"
211         "movq 7*8+0*8(%%"REG_S"), %%mm0   \n\t"
212         "pmullw %%mm7, %%mm6              \n\t"
213
214         "movq %%mm4, 4*8(%%"REG_D")       \n\t"
215         "movq 7*8+1*8(%%"REG_S"), %%mm1   \n\t"
216         "pmullw %%mm7, %%mm0              \n\t"
217
218         "movq %%mm5, 5*8(%%"REG_D")       \n\t"
219         "movq 7*8+2*8(%%"REG_S"), %%mm2   \n\t"
220         "pmullw %%mm7, %%mm1              \n\t"
221
222         "movq %%mm6, 6*8(%%"REG_D")       \n\t"
223         "movq 7*8+3*8(%%"REG_S"), %%mm3   \n\t"
224         "pmullw %%mm7, %%mm2              \n\t"
225
226         "movq %%mm0, 7*8+0*8(%%"REG_D")   \n\t"
227         "movq 7*8+4*8(%%"REG_S"), %%mm4   \n\t"
228         "pmullw %%mm7, %%mm3              \n\t"
229
230         "movq %%mm1, 7*8+1*8(%%"REG_D")   \n\t"
231         "movq 7*8+5*8(%%"REG_S"), %%mm5   \n\t"
232         "pmullw %%mm7, %%mm4              \n\t"
233
234         "movq %%mm2, 7*8+2*8(%%"REG_D")   \n\t"
235         "movq 7*8+6*8(%%"REG_S"), %%mm6   \n\t"
236         "pmullw %%mm7, %%mm5              \n\t"
237
238         "movq %%mm3, 7*8+3*8(%%"REG_D")   \n\t"
239         "movq 14*8+0*8(%%"REG_S"), %%mm0  \n\t"
240         "pmullw %%mm7, %%mm6              \n\t"
241
242         "movq %%mm4, 7*8+4*8(%%"REG_D")   \n\t"
243         "movq 14*8+1*8(%%"REG_S"), %%mm1  \n\t"
244         "pmullw %%mm7, %%mm0              \n\t"
245
246         "movq %%mm5, 7*8+5*8(%%"REG_D")   \n\t"
247         "pmullw %%mm7, %%mm1              \n\t"
248
249         "movq %%mm6, 7*8+6*8(%%"REG_D")   \n\t"
250         "movq %%mm0, 14*8+0*8(%%"REG_D")  \n\t"
251         "movq %%mm1, 14*8+1*8(%%"REG_D")  \n\t"
252
253         : "+g" (q), "+S" (adr), "+D" (adr)
254         :
255         );
256 }
257
258 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_0_382683433)   = FIX64(0.382683433, 14);
259 DECLARE_ALIGNED  (8, uint64_t, ff_MM_FIX_0_541196100)= FIX64(0.541196100, 14);
260 DECLARE_ALIGNED  (8, uint64_t, ff_MM_FIX_0_707106781)= FIX64(0.707106781, 14);
261 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_1_306562965)   = FIX64(1.306562965, 14);
262
263 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_1_414213562_A) = FIX64(1.414213562, 14);
264
265 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_1_847759065)   = FIX64(1.847759065, 13);
266 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_2_613125930)   = FIX64(-2.613125930, 13);
267 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_1_414213562)   = FIX64(1.414213562, 13);
268 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_1_082392200)   = FIX64(1.082392200, 13);
269 //for t3,t5,t7 == 0 shortcut
270 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_0_847759065)   = FIX64(0.847759065, 14);
271 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_0_566454497)   = FIX64(0.566454497, 14);
272 DECLARE_ASM_CONST(8, uint64_t, MM_FIX_0_198912367)   = FIX64(0.198912367, 14);
273
274 DECLARE_ASM_CONST(8, uint64_t, MM_DESCALE_RND)       = C64(4);
275 DECLARE_ASM_CONST(8, uint64_t, MM_2)                 = C64(2);
276
277 static void column_fidct_mmx(int16_t *thr_adr, int16_t *data, int16_t *output, int cnt)
278 {
279     DECLARE_ALIGNED(8, uint64_t, temps)[4];
280
281     __asm__ volatile(
282
283         "1:                                       \n\t"
284         "movq "DCTSIZE_S"*0*2(%%"REG_S"), %%mm1   \n\t"
285         //
286         "movq "DCTSIZE_S"*3*2(%%"REG_S"), %%mm7   \n\t"
287         "movq %%mm1, %%mm0                        \n\t"
288
289         "paddw "DCTSIZE_S"*7*2(%%"REG_S"), %%mm1  \n\t" //t0
290         "movq %%mm7, %%mm3                        \n\t"
291
292         "paddw "DCTSIZE_S"*4*2(%%"REG_S"), %%mm7  \n\t" //t3
293         "movq %%mm1, %%mm5             \n\t"
294
295         "movq "DCTSIZE_S"*1*2(%%"REG_S"), %%mm6   \n\t"
296         "psubw %%mm7, %%mm1                       \n\t" //t13
297
298         "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2   \n\t"
299         "movq %%mm6, %%mm4                        \n\t"
300
301         "paddw "DCTSIZE_S"*6*2(%%"REG_S"), %%mm6  \n\t" //t1
302         "paddw %%mm7, %%mm5                       \n\t" //t10
303
304         "paddw "DCTSIZE_S"*5*2(%%"REG_S"), %%mm2  \n\t" //t2
305         "movq %%mm6, %%mm7                        \n\t"
306
307         "paddw %%mm2, %%mm6                       \n\t" //t11
308         "psubw %%mm2, %%mm7                       \n\t" //t12
309
310         "movq %%mm5, %%mm2                        \n\t"
311         "paddw %%mm6, %%mm5                       \n\t" //d0
312         // i0 t13 t12 i3 i1 d0 - d4
313         "psubw %%mm6, %%mm2                       \n\t" //d4
314         "paddw %%mm1, %%mm7                       \n\t"
315
316         "movq  4*16(%%"REG_d"), %%mm6             \n\t"
317         "psllw $2, %%mm7                          \n\t"
318
319         "psubw 0*16(%%"REG_d"), %%mm5             \n\t"
320         "psubw %%mm6, %%mm2                       \n\t"
321
322         "paddusw 0*16(%%"REG_d"), %%mm5           \n\t"
323         "paddusw %%mm6, %%mm2                     \n\t"
324
325         "pmulhw "MANGLE(ff_MM_FIX_0_707106781)", %%mm7 \n\t"
326         //
327         "paddw 0*16(%%"REG_d"), %%mm5             \n\t"
328         "paddw %%mm6, %%mm2                       \n\t"
329
330         "psubusw 0*16(%%"REG_d"), %%mm5           \n\t"
331         "psubusw %%mm6, %%mm2                     \n\t"
332
333 //This func is totally compute-bound,  operates at huge speed. So,  DC shortcut
334 // at this place isn't worthwhile due to BTB miss penalty (checked on Pent. 3).
335 //However,  typical numbers: nondc - 29%%,  dc - 46%%,  zero - 25%%. All <> 0 case is very rare.
336         "paddw "MANGLE(MM_2)", %%mm5              \n\t"
337         "movq %%mm2, %%mm6                        \n\t"
338
339         "paddw %%mm5, %%mm2                       \n\t"
340         "psubw %%mm6, %%mm5                       \n\t"
341
342         "movq %%mm1, %%mm6                        \n\t"
343         "paddw %%mm7, %%mm1                       \n\t" //d2
344
345         "psubw 2*16(%%"REG_d"), %%mm1             \n\t"
346         "psubw %%mm7, %%mm6                       \n\t" //d6
347
348         "movq 6*16(%%"REG_d"), %%mm7              \n\t"
349         "psraw $2, %%mm5                          \n\t"
350
351         "paddusw 2*16(%%"REG_d"), %%mm1           \n\t"
352         "psubw %%mm7, %%mm6                       \n\t"
353         // t7 d2 /t11 t4 t6 - d6 /t10
354
355         "paddw 2*16(%%"REG_d"), %%mm1             \n\t"
356         "paddusw %%mm7, %%mm6                     \n\t"
357
358         "psubusw 2*16(%%"REG_d"), %%mm1           \n\t"
359         "paddw %%mm7, %%mm6                       \n\t"
360
361         "psubw "DCTSIZE_S"*4*2(%%"REG_S"), %%mm3  \n\t"
362         "psubusw %%mm7, %%mm6                     \n\t"
363
364         //movq [edi+"DCTSIZE_S"*2*2], mm1
365         //movq [edi+"DCTSIZE_S"*6*2], mm6
366         "movq %%mm1, %%mm7                        \n\t"
367         "psraw $2, %%mm2                          \n\t"
368
369         "psubw "DCTSIZE_S"*6*2(%%"REG_S"), %%mm4  \n\t"
370         "psubw %%mm6, %%mm1                       \n\t"
371
372         "psubw "DCTSIZE_S"*7*2(%%"REG_S"), %%mm0  \n\t"
373         "paddw %%mm7, %%mm6                       \n\t" //'t13
374
375         "psraw $2, %%mm6                          \n\t" //paddw mm6, MM_2 !!    ---
376         "movq %%mm2, %%mm7                        \n\t"
377
378         "pmulhw "MANGLE(MM_FIX_1_414213562_A)", %%mm1 \n\t"
379         "paddw %%mm6, %%mm2                       \n\t" //'t0
380
381         "movq %%mm2, 0*8+%3                       \n\t" //!
382         "psubw %%mm6, %%mm7                       \n\t" //'t3
383
384         "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2   \n\t"
385         "psubw %%mm6, %%mm1                       \n\t" //'t12
386
387         "psubw "DCTSIZE_S"*5*2(%%"REG_S"), %%mm2  \n\t" //t5
388         "movq %%mm5, %%mm6                        \n\t"
389
390         "movq %%mm7, 3*8+%3                       \n\t"
391         "paddw %%mm2, %%mm3                       \n\t" //t10
392
393         "paddw %%mm4, %%mm2                       \n\t" //t11
394         "paddw %%mm0, %%mm4                       \n\t" //t12
395
396         "movq %%mm3, %%mm7                        \n\t"
397         "psubw %%mm4, %%mm3                       \n\t"
398
399         "psllw $2, %%mm3                          \n\t"
400         "psllw $2, %%mm7                          \n\t" //opt for P6
401
402         "pmulhw "MANGLE(MM_FIX_0_382683433)", %%mm3 \n\t"
403         "psllw $2, %%mm4                          \n\t"
404
405         "pmulhw "MANGLE(ff_MM_FIX_0_541196100)", %%mm7 \n\t"
406         "psllw $2, %%mm2                          \n\t"
407
408         "pmulhw "MANGLE(MM_FIX_1_306562965)", %%mm4 \n\t"
409         "paddw %%mm1, %%mm5                       \n\t" //'t1
410
411         "pmulhw "MANGLE(ff_MM_FIX_0_707106781)", %%mm2 \n\t"
412         "psubw %%mm1, %%mm6                       \n\t" //'t2
413         // t7 't12 't11 t4 t6 - 't13 't10   ---
414
415         "paddw %%mm3, %%mm7                       \n\t" //z2
416
417         "movq %%mm5, 1*8+%3                       \n\t"
418         "paddw %%mm3, %%mm4                       \n\t" //z4
419
420         "movq 3*16(%%"REG_d"), %%mm3              \n\t"
421         "movq %%mm0, %%mm1                        \n\t"
422
423         "movq %%mm6, 2*8+%3                       \n\t"
424         "psubw %%mm2, %%mm1                       \n\t" //z13
425
426 //===
427         "paddw %%mm2, %%mm0                       \n\t" //z11
428         "movq %%mm1, %%mm5                        \n\t"
429
430         "movq 5*16(%%"REG_d"), %%mm2              \n\t"
431         "psubw %%mm7, %%mm1                       \n\t" //d3
432
433         "paddw %%mm7, %%mm5                       \n\t" //d5
434         "psubw %%mm3, %%mm1                       \n\t"
435
436         "movq 1*16(%%"REG_d"), %%mm7              \n\t"
437         "psubw %%mm2, %%mm5                       \n\t"
438
439         "movq %%mm0, %%mm6                        \n\t"
440         "paddw %%mm4, %%mm0                       \n\t" //d1
441
442         "paddusw %%mm3, %%mm1                     \n\t"
443         "psubw %%mm4, %%mm6                       \n\t" //d7
444
445         // d1 d3 - - - d5 d7 -
446         "movq 7*16(%%"REG_d"), %%mm4              \n\t"
447         "psubw %%mm7, %%mm0                       \n\t"
448
449         "psubw %%mm4, %%mm6                       \n\t"
450         "paddusw %%mm2, %%mm5                     \n\t"
451
452         "paddusw %%mm4, %%mm6                     \n\t"
453         "paddw %%mm3, %%mm1                       \n\t"
454
455         "paddw %%mm2, %%mm5                       \n\t"
456         "paddw %%mm4, %%mm6                       \n\t"
457
458         "psubusw %%mm3, %%mm1                     \n\t"
459         "psubusw %%mm2, %%mm5                     \n\t"
460
461         "psubusw %%mm4, %%mm6                     \n\t"
462         "movq %%mm1, %%mm4                        \n\t"
463
464         "por %%mm5, %%mm4                         \n\t"
465         "paddusw %%mm7, %%mm0                     \n\t"
466
467         "por %%mm6, %%mm4                         \n\t"
468         "paddw %%mm7, %%mm0                       \n\t"
469
470         "packssdw %%mm4, %%mm4                    \n\t"
471         "psubusw %%mm7, %%mm0                     \n\t"
472
473         "movd %%mm4, %%"REG_a"                    \n\t"
474         "or %%"REG_a", %%"REG_a"                  \n\t"
475         "jnz 2f                                   \n\t"
476         //movq [edi+"DCTSIZE_S"*3*2], mm1
477         //movq [edi+"DCTSIZE_S"*5*2], mm5
478         //movq [edi+"DCTSIZE_S"*1*2], mm0
479         //movq [edi+"DCTSIZE_S"*7*2], mm6
480         // t4 t5 - - - t6 t7 -
481         //--- t4 (mm0) may be <>0; mm1, mm5, mm6 == 0
482 //Typical numbers: nondc - 19%%,  dc - 26%%,  zero - 55%%. zero case alone isn't worthwhile
483         "movq 0*8+%3, %%mm4                      \n\t"
484         "movq %%mm0, %%mm1                       \n\t"
485
486         "pmulhw "MANGLE(MM_FIX_0_847759065)", %%mm0 \n\t" //tmp6
487         "movq %%mm1, %%mm2                       \n\t"
488
489         "movq "DCTSIZE_S"*0*2(%%"REG_D"), %%mm5 \n\t"
490         "movq %%mm2, %%mm3                      \n\t"
491
492         "pmulhw "MANGLE(MM_FIX_0_566454497)", %%mm1 \n\t" //tmp5
493         "paddw %%mm4, %%mm5                     \n\t"
494
495         "movq 1*8+%3, %%mm6                     \n\t"
496         //paddw mm3, MM_2
497         "psraw $2, %%mm3                        \n\t" //tmp7
498
499         "pmulhw "MANGLE(MM_FIX_0_198912367)", %%mm2 \n\t" //-tmp4
500         "psubw %%mm3, %%mm4                     \n\t"
501
502         "movq "DCTSIZE_S"*1*2(%%"REG_D"), %%mm7 \n\t"
503         "paddw %%mm3, %%mm5                     \n\t"
504
505         "movq %%mm4, "DCTSIZE_S"*7*2(%%"REG_D") \n\t"
506         "paddw %%mm6, %%mm7                     \n\t"
507
508         "movq 2*8+%3, %%mm3                     \n\t"
509         "psubw %%mm0, %%mm6                     \n\t"
510
511         "movq "DCTSIZE_S"*2*2(%%"REG_D"), %%mm4 \n\t"
512         "paddw %%mm0, %%mm7                     \n\t"
513
514         "movq %%mm5, "DCTSIZE_S"*0*2(%%"REG_D") \n\t"
515         "paddw %%mm3, %%mm4                     \n\t"
516
517         "movq %%mm6, "DCTSIZE_S"*6*2(%%"REG_D") \n\t"
518         "psubw %%mm1, %%mm3                     \n\t"
519
520         "movq "DCTSIZE_S"*5*2(%%"REG_D"), %%mm5 \n\t"
521         "paddw %%mm1, %%mm4                     \n\t"
522
523         "movq "DCTSIZE_S"*3*2(%%"REG_D"), %%mm6 \n\t"
524         "paddw %%mm3, %%mm5                     \n\t"
525
526         "movq 3*8+%3, %%mm0                     \n\t"
527         "add $8, %%"REG_S"                      \n\t"
528
529         "movq %%mm7, "DCTSIZE_S"*1*2(%%"REG_D") \n\t"
530         "paddw %%mm0, %%mm6                     \n\t"
531
532         "movq %%mm4, "DCTSIZE_S"*2*2(%%"REG_D") \n\t"
533         "psubw %%mm2, %%mm0                     \n\t"
534
535         "movq "DCTSIZE_S"*4*2(%%"REG_D"), %%mm7 \n\t"
536         "paddw %%mm2, %%mm6                     \n\t"
537
538         "movq %%mm5, "DCTSIZE_S"*5*2(%%"REG_D") \n\t"
539         "paddw %%mm0, %%mm7                     \n\t"
540
541         "movq %%mm6, "DCTSIZE_S"*3*2(%%"REG_D") \n\t"
542
543         "movq %%mm7, "DCTSIZE_S"*4*2(%%"REG_D") \n\t"
544         "add $8, %%"REG_D"                      \n\t"
545         "jmp 4f                                 \n\t"
546
547         "2:                                     \n\t"
548         //--- non DC2
549         //psraw mm1, 2 w/o it -> offset. thr1, thr1, thr1  (actually thr1, thr1, thr1-1)
550         //psraw mm5, 2
551         //psraw mm0, 2
552         //psraw mm6, 2
553         "movq %%mm5, %%mm3                      \n\t"
554         "psubw %%mm1, %%mm5                     \n\t"
555
556         "psllw $1, %%mm5                        \n\t" //'z10
557         "paddw %%mm1, %%mm3                     \n\t" //'z13
558
559         "movq %%mm0, %%mm2                      \n\t"
560         "psubw %%mm6, %%mm0                     \n\t"
561
562         "movq %%mm5, %%mm1                      \n\t"
563         "psllw $1, %%mm0                        \n\t" //'z12
564
565         "pmulhw "MANGLE(MM_FIX_2_613125930)", %%mm1 \n\t" //-
566         "paddw %%mm0, %%mm5                     \n\t"
567
568         "pmulhw "MANGLE(MM_FIX_1_847759065)", %%mm5 \n\t" //'z5
569         "paddw %%mm6, %%mm2                     \n\t" //'z11
570
571         "pmulhw "MANGLE(MM_FIX_1_082392200)", %%mm0 \n\t"
572         "movq %%mm2, %%mm7                      \n\t"
573
574         //---
575         "movq 0*8+%3, %%mm4                     \n\t"
576         "psubw %%mm3, %%mm2                     \n\t"
577
578         "psllw $1, %%mm2                        \n\t"
579         "paddw %%mm3, %%mm7                     \n\t" //'t7
580
581         "pmulhw "MANGLE(MM_FIX_1_414213562)", %%mm2 \n\t" //'t11
582         "movq %%mm4, %%mm6                      \n\t"
583         //paddw mm7, MM_2
584         "psraw $2, %%mm7                        \n\t"
585
586         "paddw "DCTSIZE_S"*0*2(%%"REG_D"), %%mm4\n\t"
587         "psubw %%mm7, %%mm6                     \n\t"
588
589         "movq 1*8+%3, %%mm3                     \n\t"
590         "paddw %%mm7, %%mm4                     \n\t"
591
592         "movq %%mm6, "DCTSIZE_S"*7*2(%%"REG_D") \n\t"
593         "paddw %%mm5, %%mm1                     \n\t" //'t12
594
595         "movq %%mm4, "DCTSIZE_S"*0*2(%%"REG_D") \n\t"
596         "psubw %%mm7, %%mm1                     \n\t" //'t6
597
598         "movq 2*8+%3, %%mm7                     \n\t"
599         "psubw %%mm5, %%mm0                     \n\t" //'t10
600
601         "movq 3*8+%3, %%mm6                     \n\t"
602         "movq %%mm3, %%mm5                      \n\t"
603
604         "paddw "DCTSIZE_S"*1*2(%%"REG_D"), %%mm3\n\t"
605         "psubw %%mm1, %%mm5                     \n\t"
606
607         "psubw %%mm1, %%mm2                     \n\t" //'t5
608         "paddw %%mm1, %%mm3                     \n\t"
609
610         "movq %%mm5, "DCTSIZE_S"*6*2(%%"REG_D") \n\t"
611         "movq %%mm7, %%mm4                      \n\t"
612
613         "paddw "DCTSIZE_S"*2*2(%%"REG_D"), %%mm7\n\t"
614         "psubw %%mm2, %%mm4                     \n\t"
615
616         "paddw "DCTSIZE_S"*5*2(%%"REG_D"), %%mm4\n\t"
617         "paddw %%mm2, %%mm7                     \n\t"
618
619         "movq %%mm3, "DCTSIZE_S"*1*2(%%"REG_D") \n\t"
620         "paddw %%mm2, %%mm0                     \n\t" //'t4
621
622         // 't4 't6 't5 - - - - 't7
623         "movq %%mm7, "DCTSIZE_S"*2*2(%%"REG_D") \n\t"
624         "movq %%mm6, %%mm1                      \n\t"
625
626         "paddw "DCTSIZE_S"*4*2(%%"REG_D"), %%mm6\n\t"
627         "psubw %%mm0, %%mm1                     \n\t"
628
629         "paddw "DCTSIZE_S"*3*2(%%"REG_D"), %%mm1\n\t"
630         "paddw %%mm0, %%mm6                     \n\t"
631
632         "movq %%mm4, "DCTSIZE_S"*5*2(%%"REG_D") \n\t"
633         "add $8, %%"REG_S"                      \n\t"
634
635         "movq %%mm6, "DCTSIZE_S"*4*2(%%"REG_D") \n\t"
636
637         "movq %%mm1, "DCTSIZE_S"*3*2(%%"REG_D") \n\t"
638         "add $8, %%"REG_D"                      \n\t"
639
640         "4:                                     \n\t"
641         "movq "DCTSIZE_S"*0*2(%%"REG_S"), %%mm1 \n\t"
642         //
643         "movq "DCTSIZE_S"*3*2(%%"REG_S"), %%mm7 \n\t"
644         "movq %%mm1, %%mm0                      \n\t"
645
646         "paddw "DCTSIZE_S"*7*2(%%"REG_S"), %%mm1\n\t" //t0
647         "movq %%mm7, %%mm3                      \n\t"
648
649         "paddw "DCTSIZE_S"*4*2(%%"REG_S"), %%mm7\n\t" //t3
650         "movq %%mm1, %%mm5                      \n\t"
651
652         "movq "DCTSIZE_S"*1*2(%%"REG_S"), %%mm6 \n\t"
653         "psubw %%mm7, %%mm1                     \n\t" //t13
654
655         "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2 \n\t"
656         "movq %%mm6, %%mm4                      \n\t"
657
658         "paddw "DCTSIZE_S"*6*2(%%"REG_S"), %%mm6\n\t" //t1
659         "paddw %%mm7, %%mm5                     \n\t" //t10
660
661         "paddw "DCTSIZE_S"*5*2(%%"REG_S"), %%mm2\n\t" //t2
662         "movq %%mm6, %%mm7                      \n\t"
663
664         "paddw %%mm2, %%mm6                     \n\t" //t11
665         "psubw %%mm2, %%mm7                     \n\t" //t12
666
667         "movq %%mm5, %%mm2                      \n\t"
668         "paddw %%mm6, %%mm5                     \n\t" //d0
669         // i0 t13 t12 i3 i1 d0 - d4
670         "psubw %%mm6, %%mm2                     \n\t" //d4
671         "paddw %%mm1, %%mm7                     \n\t"
672
673         "movq  1*8+4*16(%%"REG_d"), %%mm6       \n\t"
674         "psllw $2, %%mm7                        \n\t"
675
676         "psubw 1*8+0*16(%%"REG_d"), %%mm5       \n\t"
677         "psubw %%mm6, %%mm2                     \n\t"
678
679         "paddusw 1*8+0*16(%%"REG_d"), %%mm5     \n\t"
680         "paddusw %%mm6, %%mm2                   \n\t"
681
682         "pmulhw "MANGLE(ff_MM_FIX_0_707106781)", %%mm7 \n\t"
683         //
684         "paddw 1*8+0*16(%%"REG_d"), %%mm5       \n\t"
685         "paddw %%mm6, %%mm2                     \n\t"
686
687         "psubusw 1*8+0*16(%%"REG_d"), %%mm5     \n\t"
688         "psubusw %%mm6, %%mm2                   \n\t"
689
690 //This func is totally compute-bound,  operates at huge speed. So,  DC shortcut
691 // at this place isn't worthwhile due to BTB miss penalty (checked on Pent. 3).
692 //However,  typical numbers: nondc - 29%%,  dc - 46%%,  zero - 25%%. All <> 0 case is very rare.
693         "paddw "MANGLE(MM_2)", %%mm5            \n\t"
694         "movq %%mm2, %%mm6                      \n\t"
695
696         "paddw %%mm5, %%mm2                     \n\t"
697         "psubw %%mm6, %%mm5                     \n\t"
698
699         "movq %%mm1, %%mm6                      \n\t"
700         "paddw %%mm7, %%mm1                     \n\t" //d2
701
702         "psubw 1*8+2*16(%%"REG_d"), %%mm1       \n\t"
703         "psubw %%mm7, %%mm6                     \n\t" //d6
704
705         "movq 1*8+6*16(%%"REG_d"), %%mm7        \n\t"
706         "psraw $2, %%mm5                        \n\t"
707
708         "paddusw 1*8+2*16(%%"REG_d"), %%mm1     \n\t"
709         "psubw %%mm7, %%mm6                     \n\t"
710         // t7 d2 /t11 t4 t6 - d6 /t10
711
712         "paddw 1*8+2*16(%%"REG_d"), %%mm1       \n\t"
713         "paddusw %%mm7, %%mm6                   \n\t"
714
715         "psubusw 1*8+2*16(%%"REG_d"), %%mm1     \n\t"
716         "paddw %%mm7, %%mm6                     \n\t"
717
718         "psubw "DCTSIZE_S"*4*2(%%"REG_S"), %%mm3\n\t"
719         "psubusw %%mm7, %%mm6                   \n\t"
720
721         //movq [edi+"DCTSIZE_S"*2*2], mm1
722         //movq [edi+"DCTSIZE_S"*6*2], mm6
723         "movq %%mm1, %%mm7                      \n\t"
724         "psraw $2, %%mm2                        \n\t"
725
726         "psubw "DCTSIZE_S"*6*2(%%"REG_S"), %%mm4\n\t"
727         "psubw %%mm6, %%mm1                     \n\t"
728
729         "psubw "DCTSIZE_S"*7*2(%%"REG_S"), %%mm0\n\t"
730         "paddw %%mm7, %%mm6                     \n\t" //'t13
731
732         "psraw $2, %%mm6                        \n\t" //paddw mm6, MM_2 !!    ---
733         "movq %%mm2, %%mm7                      \n\t"
734
735         "pmulhw "MANGLE(MM_FIX_1_414213562_A)", %%mm1 \n\t"
736         "paddw %%mm6, %%mm2                     \n\t" //'t0
737
738         "movq %%mm2, 0*8+%3                     \n\t" //!
739         "psubw %%mm6, %%mm7                     \n\t" //'t3
740
741         "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2 \n\t"
742         "psubw %%mm6, %%mm1                     \n\t" //'t12
743
744         "psubw "DCTSIZE_S"*5*2(%%"REG_S"), %%mm2\n\t" //t5
745         "movq %%mm5, %%mm6                      \n\t"
746
747         "movq %%mm7, 3*8+%3                     \n\t"
748         "paddw %%mm2, %%mm3                     \n\t" //t10
749
750         "paddw %%mm4, %%mm2                     \n\t" //t11
751         "paddw %%mm0, %%mm4                     \n\t" //t12
752
753         "movq %%mm3, %%mm7                      \n\t"
754         "psubw %%mm4, %%mm3                     \n\t"
755
756         "psllw $2, %%mm3                        \n\t"
757         "psllw $2, %%mm7                        \n\t" //opt for P6
758
759         "pmulhw "MANGLE(MM_FIX_0_382683433)", %%mm3 \n\t"
760         "psllw $2, %%mm4                        \n\t"
761
762         "pmulhw "MANGLE(ff_MM_FIX_0_541196100)", %%mm7 \n\t"
763         "psllw $2, %%mm2                        \n\t"
764
765         "pmulhw "MANGLE(MM_FIX_1_306562965)", %%mm4 \n\t"
766         "paddw %%mm1, %%mm5                     \n\t" //'t1
767
768         "pmulhw "MANGLE(ff_MM_FIX_0_707106781)", %%mm2 \n\t"
769         "psubw %%mm1, %%mm6                     \n\t" //'t2
770         // t7 't12 't11 t4 t6 - 't13 't10   ---
771
772         "paddw %%mm3, %%mm7                     \n\t" //z2
773
774         "movq %%mm5, 1*8+%3                     \n\t"
775         "paddw %%mm3, %%mm4                     \n\t" //z4
776
777         "movq 1*8+3*16(%%"REG_d"), %%mm3        \n\t"
778         "movq %%mm0, %%mm1                      \n\t"
779
780         "movq %%mm6, 2*8+%3                     \n\t"
781         "psubw %%mm2, %%mm1                     \n\t" //z13
782
783 //===
784         "paddw %%mm2, %%mm0                     \n\t" //z11
785         "movq %%mm1, %%mm5                      \n\t"
786
787         "movq 1*8+5*16(%%"REG_d"), %%mm2        \n\t"
788         "psubw %%mm7, %%mm1                     \n\t" //d3
789
790         "paddw %%mm7, %%mm5                     \n\t" //d5
791         "psubw %%mm3, %%mm1                     \n\t"
792
793         "movq 1*8+1*16(%%"REG_d"), %%mm7        \n\t"
794         "psubw %%mm2, %%mm5                     \n\t"
795
796         "movq %%mm0, %%mm6                      \n\t"
797         "paddw %%mm4, %%mm0                     \n\t" //d1
798
799         "paddusw %%mm3, %%mm1                   \n\t"
800         "psubw %%mm4, %%mm6                     \n\t" //d7
801
802         // d1 d3 - - - d5 d7 -
803         "movq 1*8+7*16(%%"REG_d"), %%mm4        \n\t"
804         "psubw %%mm7, %%mm0                     \n\t"
805
806         "psubw %%mm4, %%mm6                     \n\t"
807         "paddusw %%mm2, %%mm5                   \n\t"
808
809         "paddusw %%mm4, %%mm6                   \n\t"
810         "paddw %%mm3, %%mm1                     \n\t"
811
812         "paddw %%mm2, %%mm5                     \n\t"
813         "paddw %%mm4, %%mm6                     \n\t"
814
815         "psubusw %%mm3, %%mm1                   \n\t"
816         "psubusw %%mm2, %%mm5                   \n\t"
817
818         "psubusw %%mm4, %%mm6                   \n\t"
819         "movq %%mm1, %%mm4                      \n\t"
820
821         "por %%mm5, %%mm4                       \n\t"
822         "paddusw %%mm7, %%mm0                   \n\t"
823
824         "por %%mm6, %%mm4                       \n\t"
825         "paddw %%mm7, %%mm0                     \n\t"
826
827         "packssdw %%mm4, %%mm4                  \n\t"
828         "psubusw %%mm7, %%mm0                   \n\t"
829
830         "movd %%mm4, %%"REG_a"                  \n\t"
831         "or %%"REG_a", %%"REG_a"                \n\t"
832         "jnz 3f                                 \n\t"
833         //movq [edi+"DCTSIZE_S"*3*2], mm1
834         //movq [edi+"DCTSIZE_S"*5*2], mm5
835         //movq [edi+"DCTSIZE_S"*1*2], mm0
836         //movq [edi+"DCTSIZE_S"*7*2], mm6
837         // t4 t5 - - - t6 t7 -
838         //--- t4 (mm0) may be <>0; mm1, mm5, mm6 == 0
839 //Typical numbers: nondc - 19%%,  dc - 26%%,  zero - 55%%. zero case alone isn't worthwhile
840         "movq 0*8+%3, %%mm4                    \n\t"
841         "movq %%mm0, %%mm1                     \n\t"
842
843         "pmulhw "MANGLE(MM_FIX_0_847759065)", %%mm0 \n\t" //tmp6
844         "movq %%mm1, %%mm2                     \n\t"
845
846         "movq "DCTSIZE_S"*0*2(%%"REG_D"), %%mm5\n\t"
847         "movq %%mm2, %%mm3                     \n\t"
848
849         "pmulhw "MANGLE(MM_FIX_0_566454497)", %%mm1 \n\t" //tmp5
850         "paddw %%mm4, %%mm5                    \n\t"
851
852         "movq 1*8+%3, %%mm6                    \n\t"
853         //paddw mm3, MM_2
854         "psraw $2, %%mm3                       \n\t" //tmp7
855
856         "pmulhw "MANGLE(MM_FIX_0_198912367)", %%mm2 \n\t" //-tmp4
857         "psubw %%mm3, %%mm4                    \n\t"
858
859         "movq "DCTSIZE_S"*1*2(%%"REG_D"), %%mm7\n\t"
860         "paddw %%mm3, %%mm5                    \n\t"
861
862         "movq %%mm4, "DCTSIZE_S"*7*2(%%"REG_D")\n\t"
863         "paddw %%mm6, %%mm7                    \n\t"
864
865         "movq 2*8+%3, %%mm3                    \n\t"
866         "psubw %%mm0, %%mm6                    \n\t"
867
868         "movq "DCTSIZE_S"*2*2(%%"REG_D"), %%mm4\n\t"
869         "paddw %%mm0, %%mm7                    \n\t"
870
871         "movq %%mm5, "DCTSIZE_S"*0*2(%%"REG_D")\n\t"
872         "paddw %%mm3, %%mm4                    \n\t"
873
874         "movq %%mm6, "DCTSIZE_S"*6*2(%%"REG_D")\n\t"
875         "psubw %%mm1, %%mm3                    \n\t"
876
877         "movq "DCTSIZE_S"*5*2(%%"REG_D"), %%mm5\n\t"
878         "paddw %%mm1, %%mm4                    \n\t"
879
880         "movq "DCTSIZE_S"*3*2(%%"REG_D"), %%mm6\n\t"
881         "paddw %%mm3, %%mm5                    \n\t"
882
883         "movq 3*8+%3, %%mm0                    \n\t"
884         "add $24, %%"REG_S"                    \n\t"
885
886         "movq %%mm7, "DCTSIZE_S"*1*2(%%"REG_D")\n\t"
887         "paddw %%mm0, %%mm6                    \n\t"
888
889         "movq %%mm4, "DCTSIZE_S"*2*2(%%"REG_D")\n\t"
890         "psubw %%mm2, %%mm0                    \n\t"
891
892         "movq "DCTSIZE_S"*4*2(%%"REG_D"), %%mm7\n\t"
893         "paddw %%mm2, %%mm6                    \n\t"
894
895         "movq %%mm5, "DCTSIZE_S"*5*2(%%"REG_D")\n\t"
896         "paddw %%mm0, %%mm7                    \n\t"
897
898         "movq %%mm6, "DCTSIZE_S"*3*2(%%"REG_D")\n\t"
899
900         "movq %%mm7, "DCTSIZE_S"*4*2(%%"REG_D")\n\t"
901         "add $24, %%"REG_D"                    \n\t"
902         "sub $2, %%"REG_c"                     \n\t"
903         "jnz 1b                                \n\t"
904         "jmp 5f                                \n\t"
905
906         "3:                                    \n\t"
907         //--- non DC2
908         //psraw mm1, 2 w/o it -> offset. thr1, thr1, thr1  (actually thr1, thr1, thr1-1)
909         //psraw mm5, 2
910         //psraw mm0, 2
911         //psraw mm6, 2
912         "movq %%mm5, %%mm3                    \n\t"
913         "psubw %%mm1, %%mm5                   \n\t"
914
915         "psllw $1, %%mm5                      \n\t" //'z10
916         "paddw %%mm1, %%mm3                   \n\t" //'z13
917
918         "movq %%mm0, %%mm2                    \n\t"
919         "psubw %%mm6, %%mm0                   \n\t"
920
921         "movq %%mm5, %%mm1                    \n\t"
922         "psllw $1, %%mm0                      \n\t" //'z12
923
924         "pmulhw "MANGLE(MM_FIX_2_613125930)", %%mm1 \n\t" //-
925         "paddw %%mm0, %%mm5                   \n\t"
926
927         "pmulhw "MANGLE(MM_FIX_1_847759065)", %%mm5 \n\t" //'z5
928         "paddw %%mm6, %%mm2                   \n\t" //'z11
929
930         "pmulhw "MANGLE(MM_FIX_1_082392200)", %%mm0 \n\t"
931         "movq %%mm2, %%mm7                    \n\t"
932
933         //---
934         "movq 0*8+%3, %%mm4                   \n\t"
935         "psubw %%mm3, %%mm2                   \n\t"
936
937         "psllw $1, %%mm2                      \n\t"
938         "paddw %%mm3, %%mm7                   \n\t" //'t7
939
940         "pmulhw "MANGLE(MM_FIX_1_414213562)", %%mm2 \n\t" //'t11
941         "movq %%mm4, %%mm6                    \n\t"
942         //paddw mm7, MM_2
943         "psraw $2, %%mm7                      \n\t"
944
945         "paddw "DCTSIZE_S"*0*2(%%"REG_D"), %%mm4 \n\t"
946         "psubw %%mm7, %%mm6                   \n\t"
947
948         "movq 1*8+%3, %%mm3                   \n\t"
949         "paddw %%mm7, %%mm4                   \n\t"
950
951         "movq %%mm6, "DCTSIZE_S"*7*2(%%"REG_D") \n\t"
952         "paddw %%mm5, %%mm1                   \n\t" //'t12
953
954         "movq %%mm4, "DCTSIZE_S"*0*2(%%"REG_D") \n\t"
955         "psubw %%mm7, %%mm1                   \n\t" //'t6
956
957         "movq 2*8+%3, %%mm7                   \n\t"
958         "psubw %%mm5, %%mm0                   \n\t" //'t10
959
960         "movq 3*8+%3, %%mm6                   \n\t"
961         "movq %%mm3, %%mm5                    \n\t"
962
963         "paddw "DCTSIZE_S"*1*2(%%"REG_D"), %%mm3 \n\t"
964         "psubw %%mm1, %%mm5                   \n\t"
965
966         "psubw %%mm1, %%mm2                   \n\t" //'t5
967         "paddw %%mm1, %%mm3                   \n\t"
968
969         "movq %%mm5, "DCTSIZE_S"*6*2(%%"REG_D") \n\t"
970         "movq %%mm7, %%mm4                    \n\t"
971
972         "paddw "DCTSIZE_S"*2*2(%%"REG_D"), %%mm7 \n\t"
973         "psubw %%mm2, %%mm4                   \n\t"
974
975         "paddw "DCTSIZE_S"*5*2(%%"REG_D"), %%mm4 \n\t"
976         "paddw %%mm2, %%mm7                   \n\t"
977
978         "movq %%mm3, "DCTSIZE_S"*1*2(%%"REG_D") \n\t"
979         "paddw %%mm2, %%mm0                    \n\t" //'t4
980
981         // 't4 't6 't5 - - - - 't7
982         "movq %%mm7, "DCTSIZE_S"*2*2(%%"REG_D") \n\t"
983         "movq %%mm6, %%mm1                     \n\t"
984
985         "paddw "DCTSIZE_S"*4*2(%%"REG_D"), %%mm6 \n\t"
986         "psubw %%mm0, %%mm1                    \n\t"
987
988         "paddw "DCTSIZE_S"*3*2(%%"REG_D"), %%mm1 \n\t"
989         "paddw %%mm0, %%mm6                    \n\t"
990
991         "movq %%mm4, "DCTSIZE_S"*5*2(%%"REG_D") \n\t"
992         "add $24, %%"REG_S"                    \n\t"
993
994         "movq %%mm6, "DCTSIZE_S"*4*2(%%"REG_D") \n\t"
995
996         "movq %%mm1, "DCTSIZE_S"*3*2(%%"REG_D") \n\t"
997         "add $24, %%"REG_D"                    \n\t"
998         "sub $2, %%"REG_c"                     \n\t"
999         "jnz 1b                                \n\t"
1000         "5:                                    \n\t"
1001
1002         : "+S"(data), "+D"(output), "+c"(cnt), "=o"(temps)
1003         : "d"(thr_adr)
1004           NAMED_CONSTRAINTS_ADD(ff_MM_FIX_0_707106781, MM_2,MM_FIX_1_414213562_A, MM_FIX_1_414213562, MM_FIX_0_382683433,
1005                                 ff_MM_FIX_0_541196100, MM_FIX_1_306562965, MM_FIX_0_847759065)
1006           NAMED_CONSTRAINTS_ADD(MM_FIX_0_566454497, MM_FIX_0_198912367, MM_FIX_2_613125930, MM_FIX_1_847759065,
1007                                 MM_FIX_1_082392200)
1008         : "%"REG_a
1009         );
1010 }
1011
1012 static void row_idct_mmx (int16_t *workspace, int16_t *output_adr, int output_stride, int cnt)
1013 {
1014     DECLARE_ALIGNED(8, uint64_t, temps)[4];
1015
1016     __asm__ volatile(
1017         "lea (%%"REG_a",%%"REG_a",2), %%"REG_d"    \n\t"
1018         "1:                     \n\t"
1019         "movq "DCTSIZE_S"*0*2(%%"REG_S"), %%mm0    \n\t"
1020         //
1021
1022         "movq "DCTSIZE_S"*1*2(%%"REG_S"), %%mm1    \n\t"
1023         "movq %%mm0, %%mm4                         \n\t"
1024
1025         "movq "DCTSIZE_S"*2*2(%%"REG_S"), %%mm2    \n\t"
1026         "punpcklwd %%mm1, %%mm0                    \n\t"
1027
1028         "movq "DCTSIZE_S"*3*2(%%"REG_S"), %%mm3    \n\t"
1029         "punpckhwd %%mm1, %%mm4                    \n\t"
1030
1031         //transpose 4x4
1032         "movq %%mm2, %%mm7                         \n\t"
1033         "punpcklwd %%mm3, %%mm2                    \n\t"
1034
1035         "movq %%mm0, %%mm6                         \n\t"
1036         "punpckldq %%mm2, %%mm0                    \n\t" //0
1037
1038         "punpckhdq %%mm2, %%mm6                    \n\t" //1
1039         "movq %%mm0, %%mm5                         \n\t"
1040
1041         "punpckhwd %%mm3, %%mm7                    \n\t"
1042         "psubw %%mm6, %%mm0                        \n\t"
1043
1044         "pmulhw "MANGLE(MM_FIX_1_414213562_A)", %%mm0 \n\t"
1045         "movq %%mm4, %%mm2                         \n\t"
1046
1047         "punpckldq %%mm7, %%mm4                    \n\t" //2
1048         "paddw %%mm6, %%mm5                        \n\t"
1049
1050         "punpckhdq %%mm7, %%mm2                    \n\t" //3
1051         "movq %%mm4, %%mm1                         \n\t"
1052
1053         "psllw $2, %%mm0                           \n\t"
1054         "paddw %%mm2, %%mm4                        \n\t" //t10
1055
1056         "movq "DCTSIZE_S"*0*2+"DCTSIZE_S"(%%"REG_S"), %%mm3 \n\t"
1057         "psubw %%mm2, %%mm1                        \n\t" //t11
1058
1059         "movq "DCTSIZE_S"*1*2+"DCTSIZE_S"(%%"REG_S"), %%mm2 \n\t"
1060         "psubw %%mm5, %%mm0                        \n\t"
1061
1062         "movq %%mm4, %%mm6                         \n\t"
1063         "paddw %%mm5, %%mm4                        \n\t" //t0
1064
1065         "psubw %%mm5, %%mm6                        \n\t" //t3
1066         "movq %%mm1, %%mm7                         \n\t"
1067
1068         "movq "DCTSIZE_S"*2*2+"DCTSIZE_S"(%%"REG_S"), %%mm5 \n\t"
1069         "paddw %%mm0, %%mm1                        \n\t" //t1
1070
1071         "movq %%mm4, 0*8+%3                        \n\t" //t0
1072         "movq %%mm3, %%mm4                         \n\t"
1073
1074         "movq %%mm6, 1*8+%3                        \n\t" //t3
1075         "punpcklwd %%mm2, %%mm3                    \n\t"
1076
1077         //transpose 4x4
1078         "movq "DCTSIZE_S"*3*2+"DCTSIZE_S"(%%"REG_S"), %%mm6 \n\t"
1079         "punpckhwd %%mm2, %%mm4                    \n\t"
1080
1081         "movq %%mm5, %%mm2                         \n\t"
1082         "punpcklwd %%mm6, %%mm5                    \n\t"
1083
1084         "psubw %%mm0, %%mm7                        \n\t" //t2
1085         "punpckhwd %%mm6, %%mm2                    \n\t"
1086
1087         "movq %%mm3, %%mm0                         \n\t"
1088         "punpckldq %%mm5, %%mm3                    \n\t" //4
1089
1090         "punpckhdq %%mm5, %%mm0                    \n\t" //5
1091         "movq %%mm4, %%mm5                         \n\t"
1092
1093         //
1094         "movq %%mm3, %%mm6                         \n\t"
1095         "punpckldq %%mm2, %%mm4                    \n\t" //6
1096
1097         "psubw %%mm0, %%mm3                        \n\t" //z10
1098         "punpckhdq %%mm2, %%mm5                    \n\t" //7
1099
1100         "paddw %%mm0, %%mm6                        \n\t" //z13
1101         "movq %%mm4, %%mm2                         \n\t"
1102
1103         "movq %%mm3, %%mm0                         \n\t"
1104         "psubw %%mm5, %%mm4                        \n\t" //z12
1105
1106         "pmulhw "MANGLE(MM_FIX_2_613125930)", %%mm0\n\t" //-
1107         "paddw %%mm4, %%mm3                        \n\t"
1108
1109         "pmulhw "MANGLE(MM_FIX_1_847759065)", %%mm3\n\t" //z5
1110         "paddw %%mm5, %%mm2                        \n\t" //z11  >
1111
1112         "pmulhw "MANGLE(MM_FIX_1_082392200)", %%mm4\n\t"
1113         "movq %%mm2, %%mm5                         \n\t"
1114
1115         "psubw %%mm6, %%mm2                        \n\t"
1116         "paddw %%mm6, %%mm5                        \n\t" //t7
1117
1118         "pmulhw "MANGLE(MM_FIX_1_414213562)", %%mm2\n\t" //t11
1119         "paddw %%mm3, %%mm0                        \n\t" //t12
1120
1121         "psllw $3, %%mm0                           \n\t"
1122         "psubw %%mm3, %%mm4                        \n\t" //t10
1123
1124         "movq 0*8+%3, %%mm6                        \n\t"
1125         "movq %%mm1, %%mm3                         \n\t"
1126
1127         "psllw $3, %%mm4                           \n\t"
1128         "psubw %%mm5, %%mm0                        \n\t" //t6
1129
1130         "psllw $3, %%mm2                           \n\t"
1131         "paddw %%mm0, %%mm1                        \n\t" //d1
1132
1133         "psubw %%mm0, %%mm2                        \n\t" //t5
1134         "psubw %%mm0, %%mm3                        \n\t" //d6
1135
1136         "paddw %%mm2, %%mm4                        \n\t" //t4
1137         "movq %%mm7, %%mm0                         \n\t"
1138
1139         "paddw %%mm2, %%mm7                        \n\t" //d2
1140         "psubw %%mm2, %%mm0                        \n\t" //d5
1141
1142         "movq "MANGLE(MM_DESCALE_RND)", %%mm2      \n\t" //4
1143         "psubw %%mm5, %%mm6                        \n\t" //d7
1144
1145         "paddw 0*8+%3, %%mm5                       \n\t" //d0
1146         "paddw %%mm2, %%mm1                        \n\t"
1147
1148         "paddw %%mm2, %%mm5                        \n\t"
1149         "psraw $3, %%mm1                           \n\t"
1150
1151         "paddw %%mm2, %%mm7                        \n\t"
1152         "psraw $3, %%mm5                           \n\t"
1153
1154         "paddw (%%"REG_D"), %%mm5                  \n\t"
1155         "psraw $3, %%mm7                           \n\t"
1156
1157         "paddw (%%"REG_D",%%"REG_a"), %%mm1        \n\t"
1158         "paddw %%mm2, %%mm0                        \n\t"
1159
1160         "paddw (%%"REG_D",%%"REG_a",2), %%mm7      \n\t"
1161         "paddw %%mm2, %%mm3                        \n\t"
1162
1163         "movq %%mm5, (%%"REG_D")                   \n\t"
1164         "paddw %%mm2, %%mm6                        \n\t"
1165
1166         "movq %%mm1, (%%"REG_D",%%"REG_a")         \n\t"
1167         "psraw $3, %%mm0                           \n\t"
1168
1169         "movq %%mm7, (%%"REG_D",%%"REG_a",2)       \n\t"
1170         "add %%"REG_d", %%"REG_D"                  \n\t" //3*ls
1171
1172         "movq 1*8+%3, %%mm5                        \n\t" //t3
1173         "psraw $3, %%mm3                           \n\t"
1174
1175         "paddw (%%"REG_D",%%"REG_a",2), %%mm0      \n\t"
1176         "psubw %%mm4, %%mm5                        \n\t" //d3
1177
1178         "paddw (%%"REG_D",%%"REG_d"), %%mm3        \n\t"
1179         "psraw $3, %%mm6                           \n\t"
1180
1181         "paddw 1*8+%3, %%mm4                       \n\t" //d4
1182         "paddw %%mm2, %%mm5                        \n\t"
1183
1184         "paddw (%%"REG_D",%%"REG_a",4), %%mm6      \n\t"
1185         "paddw %%mm2, %%mm4                        \n\t"
1186
1187         "movq %%mm0, (%%"REG_D",%%"REG_a",2)       \n\t"
1188         "psraw $3, %%mm5                           \n\t"
1189
1190         "paddw (%%"REG_D"), %%mm5                  \n\t"
1191         "psraw $3, %%mm4                           \n\t"
1192
1193         "paddw (%%"REG_D",%%"REG_a"), %%mm4        \n\t"
1194         "add $"DCTSIZE_S"*2*4, %%"REG_S"           \n\t" //4 rows
1195
1196         "movq %%mm3, (%%"REG_D",%%"REG_d")         \n\t"
1197         "movq %%mm6, (%%"REG_D",%%"REG_a",4)       \n\t"
1198         "movq %%mm5, (%%"REG_D")                   \n\t"
1199         "movq %%mm4, (%%"REG_D",%%"REG_a")         \n\t"
1200
1201         "sub %%"REG_d", %%"REG_D"                  \n\t"
1202         "add $8, %%"REG_D"                         \n\t"
1203         "dec %%"REG_c"                             \n\t"
1204         "jnz 1b                                    \n\t"
1205
1206         : "+S"(workspace), "+D"(output_adr), "+c"(cnt), "=o"(temps)
1207         : "a"(output_stride * sizeof(short))
1208         NAMED_CONSTRAINTS_ADD(MM_FIX_1_414213562_A, MM_FIX_2_613125930, MM_FIX_1_847759065, MM_FIX_1_082392200,
1209                               MM_FIX_1_414213562,MM_DESCALE_RND)
1210         : "%"REG_d
1211         );
1212 }
1213
1214 static void row_fdct_mmx(int16_t *data, const uint8_t *pixels, int line_size, int cnt)
1215 {
1216     DECLARE_ALIGNED(8, uint64_t, temps)[4];
1217
1218     __asm__ volatile(
1219         "lea (%%"REG_a",%%"REG_a",2), %%"REG_d"    \n\t"
1220         "6:                                        \n\t"
1221         "movd (%%"REG_S"), %%mm0                   \n\t"
1222         "pxor %%mm7, %%mm7                         \n\t"
1223
1224         "movd (%%"REG_S",%%"REG_a"), %%mm1         \n\t"
1225         "punpcklbw %%mm7, %%mm0                    \n\t"
1226
1227         "movd (%%"REG_S",%%"REG_a",2), %%mm2       \n\t"
1228         "punpcklbw %%mm7, %%mm1                    \n\t"
1229
1230         "punpcklbw %%mm7, %%mm2                    \n\t"
1231         "add %%"REG_d", %%"REG_S"                  \n\t"
1232
1233         "movq %%mm0, %%mm5                         \n\t"
1234         //
1235
1236         "movd (%%"REG_S",%%"REG_a",4), %%mm3       \n\t" //7  ;prefetch!
1237         "movq %%mm1, %%mm6                         \n\t"
1238
1239         "movd (%%"REG_S",%%"REG_d"), %%mm4         \n\t" //6
1240         "punpcklbw %%mm7, %%mm3                    \n\t"
1241
1242         "psubw %%mm3, %%mm5                        \n\t"
1243         "punpcklbw %%mm7, %%mm4                    \n\t"
1244
1245         "paddw %%mm3, %%mm0                        \n\t"
1246         "psubw %%mm4, %%mm6                        \n\t"
1247
1248         "movd (%%"REG_S",%%"REG_a",2), %%mm3       \n\t" //5
1249         "paddw %%mm4, %%mm1                        \n\t"
1250
1251         "movq %%mm5, %3                            \n\t" //t7
1252         "punpcklbw %%mm7, %%mm3                    \n\t"
1253
1254         "movq %%mm6, %4                            \n\t" //t6
1255         "movq %%mm2, %%mm4                         \n\t"
1256
1257         "movd (%%"REG_S"), %%mm5                   \n\t" //3
1258         "paddw %%mm3, %%mm2                        \n\t"
1259
1260         "movd (%%"REG_S",%%"REG_a"), %%mm6         \n\t" //4
1261         "punpcklbw %%mm7, %%mm5                    \n\t"
1262
1263         "psubw %%mm3, %%mm4                        \n\t"
1264         "punpcklbw %%mm7, %%mm6                    \n\t"
1265
1266         "movq %%mm5, %%mm3                         \n\t"
1267         "paddw %%mm6, %%mm5                        \n\t" //t3
1268
1269         "psubw %%mm6, %%mm3                        \n\t" //t4  ; t0 t1 t2 t4 t5 t3 - -
1270         "movq %%mm0, %%mm6                         \n\t"
1271
1272         "movq %%mm1, %%mm7                         \n\t"
1273         "psubw %%mm5, %%mm0                        \n\t" //t13
1274
1275         "psubw %%mm2, %%mm1                        \n\t"
1276         "paddw %%mm2, %%mm7                        \n\t" //t11
1277
1278         "paddw %%mm0, %%mm1                        \n\t"
1279         "movq %%mm7, %%mm2                         \n\t"
1280
1281         "psllw $2, %%mm1                           \n\t"
1282         "paddw %%mm5, %%mm6                        \n\t" //t10
1283
1284         "pmulhw "MANGLE(ff_MM_FIX_0_707106781)", %%mm1 \n\t"
1285         "paddw %%mm6, %%mm7                        \n\t" //d2
1286
1287         "psubw %%mm2, %%mm6                        \n\t" //d3
1288         "movq %%mm0, %%mm5                         \n\t"
1289
1290         //transpose 4x4
1291         "movq %%mm7, %%mm2                         \n\t"
1292         "punpcklwd %%mm6, %%mm7                    \n\t"
1293
1294         "paddw %%mm1, %%mm0                        \n\t" //d0
1295         "punpckhwd %%mm6, %%mm2                    \n\t"
1296
1297         "psubw %%mm1, %%mm5                        \n\t" //d1
1298         "movq %%mm0, %%mm6                         \n\t"
1299
1300         "movq %4, %%mm1                            \n\t"
1301         "punpcklwd %%mm5, %%mm0                    \n\t"
1302
1303         "punpckhwd %%mm5, %%mm6                    \n\t"
1304         "movq %%mm0, %%mm5                         \n\t"
1305
1306         "punpckldq %%mm7, %%mm0                    \n\t" //0
1307         "paddw %%mm4, %%mm3                        \n\t"
1308
1309         "punpckhdq %%mm7, %%mm5                    \n\t" //1
1310         "movq %%mm6, %%mm7                         \n\t"
1311
1312         "movq %%mm0, "DCTSIZE_S"*0*2(%%"REG_D")    \n\t"
1313         "punpckldq %%mm2, %%mm6                    \n\t" //2
1314
1315         "movq %%mm5, "DCTSIZE_S"*1*2(%%"REG_D")    \n\t"
1316         "punpckhdq %%mm2, %%mm7                    \n\t" //3
1317
1318         "movq %%mm6, "DCTSIZE_S"*2*2(%%"REG_D")    \n\t"
1319         "paddw %%mm1, %%mm4                        \n\t"
1320
1321         "movq %%mm7, "DCTSIZE_S"*3*2(%%"REG_D")    \n\t"
1322         "psllw $2, %%mm3                           \n\t" //t10
1323
1324         "movq %3, %%mm2                            \n\t"
1325         "psllw $2, %%mm4                           \n\t" //t11
1326
1327         "pmulhw "MANGLE(ff_MM_FIX_0_707106781)", %%mm4 \n\t" //z3
1328         "paddw %%mm2, %%mm1                        \n\t"
1329
1330         "psllw $2, %%mm1                           \n\t" //t12
1331         "movq %%mm3, %%mm0                         \n\t"
1332
1333         "pmulhw "MANGLE(ff_MM_FIX_0_541196100)", %%mm0 \n\t"
1334         "psubw %%mm1, %%mm3                        \n\t"
1335
1336         "pmulhw "MANGLE(MM_FIX_0_382683433)", %%mm3 \n\t" //z5
1337         "movq %%mm2, %%mm5                         \n\t"
1338
1339         "pmulhw "MANGLE(MM_FIX_1_306562965)", %%mm1 \n\t"
1340         "psubw %%mm4, %%mm2                        \n\t" //z13
1341
1342         "paddw %%mm4, %%mm5                        \n\t" //z11
1343         "movq %%mm2, %%mm6                         \n\t"
1344
1345         "paddw %%mm3, %%mm0                        \n\t" //z2
1346         "movq %%mm5, %%mm7                         \n\t"
1347
1348         "paddw %%mm0, %%mm2                        \n\t" //d4
1349         "psubw %%mm0, %%mm6                        \n\t" //d5
1350
1351         "movq %%mm2, %%mm4                         \n\t"
1352         "paddw %%mm3, %%mm1                        \n\t" //z4
1353
1354         //transpose 4x4
1355         "punpcklwd %%mm6, %%mm2                    \n\t"
1356         "paddw %%mm1, %%mm5                        \n\t" //d6
1357
1358         "punpckhwd %%mm6, %%mm4                    \n\t"
1359         "psubw %%mm1, %%mm7                        \n\t" //d7
1360
1361         "movq %%mm5, %%mm6                         \n\t"
1362         "punpcklwd %%mm7, %%mm5                    \n\t"
1363
1364         "punpckhwd %%mm7, %%mm6                    \n\t"
1365         "movq %%mm2, %%mm7                         \n\t"
1366
1367         "punpckldq %%mm5, %%mm2                    \n\t" //4
1368         "sub %%"REG_d", %%"REG_S"                  \n\t"
1369
1370         "punpckhdq %%mm5, %%mm7                    \n\t" //5
1371         "movq %%mm4, %%mm5                         \n\t"
1372
1373         "movq %%mm2, "DCTSIZE_S"*0*2+"DCTSIZE_S"(%%"REG_D") \n\t"
1374         "punpckldq %%mm6, %%mm4                    \n\t" //6
1375
1376         "movq %%mm7, "DCTSIZE_S"*1*2+"DCTSIZE_S"(%%"REG_D") \n\t"
1377         "punpckhdq %%mm6, %%mm5                    \n\t" //7
1378
1379         "movq %%mm4, "DCTSIZE_S"*2*2+"DCTSIZE_S"(%%"REG_D") \n\t"
1380         "add $4, %%"REG_S"                         \n\t"
1381
1382         "movq %%mm5, "DCTSIZE_S"*3*2+"DCTSIZE_S"(%%"REG_D") \n\t"
1383         "add $"DCTSIZE_S"*2*4, %%"REG_D"           \n\t" //4 rows
1384         "dec %%"REG_c"                             \n\t"
1385         "jnz 6b                                    \n\t"
1386
1387         : "+S"(pixels), "+D"(data), "+c"(cnt), "=o"(temps), "=o"(temps[1])
1388         : "a"(line_size)
1389         NAMED_CONSTRAINTS_ADD(ff_MM_FIX_0_707106781, ff_MM_FIX_0_541196100, MM_FIX_0_382683433, MM_FIX_1_306562965)
1390         : "%"REG_d);
1391 }
1392
1393 av_cold void ff_fspp_init_x86(FSPPContext *s)
1394 {
1395     int cpu_flags = av_get_cpu_flags();
1396
1397     if (HAVE_MMX_INLINE && cpu_flags & AV_CPU_FLAG_MMX) {
1398         s->store_slice  = store_slice_mmx;
1399         s->store_slice2 = store_slice2_mmx;
1400         s->mul_thrmat   = mul_thrmat_mmx;
1401         s->column_fidct = column_fidct_mmx;
1402         s->row_idct     = row_idct_mmx;
1403         s->row_fdct     = row_fdct_mmx;
1404     }
1405 }