Support for MacIntel, take xx: '/nop' illegal for old versions of GAS
[ffmpeg.git] / libavcodec / i386 / simple_idct_mmx.c
1 /*
2  * Simple IDCT MMX
3  *
4  * Copyright (c) 2001, 2002 Michael Niedermayer <michaelni@gmx.at>
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 #include "../dsputil.h"
21 #include "../simple_idct.h"
22
23 /*
24 23170.475006
25 22725.260826
26 21406.727617
27 19265.545870
28 16384.000000
29 12872.826198
30 8866.956905
31 4520.335430
32 */
33 #define C0 23170 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
34 #define C1 22725 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
35 #define C2 21407 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
36 #define C3 19266 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
37 #if 0
38 #define C4 16384 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
39 #else
40 #define C4 16383 //cos(i*M_PI/16)*sqrt(2)*(1<<14) - 0.5
41 #endif
42 #define C5 12873 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
43 #define C6 8867  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
44 #define C7 4520  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
45
46 #define ROW_SHIFT 11
47 #define COL_SHIFT 20 // 6
48
49 static const uint64_t attribute_used __attribute__((aligned(8))) wm1010= 0xFFFF0000FFFF0000ULL;
50 static const uint64_t attribute_used __attribute__((aligned(8))) d40000= 0x0000000000040000ULL;
51
52 static const int16_t __attribute__((aligned(8))) coeffs[]= {
53         1<<(ROW_SHIFT-1), 0, 1<<(ROW_SHIFT-1), 0,
54 //        1<<(COL_SHIFT-1), 0, 1<<(COL_SHIFT-1), 0,
55 //        0, 1<<(COL_SHIFT-1-16), 0, 1<<(COL_SHIFT-1-16),
56         1<<(ROW_SHIFT-1), 1, 1<<(ROW_SHIFT-1), 0,
57         // the 1 = ((1<<(COL_SHIFT-1))/C4)<<ROW_SHIFT :)
58 //        0, 0, 0, 0,
59 //        0, 0, 0, 0,
60
61  C4,  C4,  C4,  C4,
62  C4, -C4,  C4, -C4,
63
64  C2,  C6,  C2,  C6,
65  C6, -C2,  C6, -C2,
66
67  C1,  C3,  C1,  C3,
68  C5,  C7,  C5,  C7,
69
70  C3, -C7,  C3, -C7,
71 -C1, -C5, -C1, -C5,
72
73  C5, -C1,  C5, -C1,
74  C7,  C3,  C7,  C3,
75
76  C7, -C5,  C7, -C5,
77  C3, -C1,  C3, -C1
78 };
79
80 #if 0
81 static void unused_var_killer(){
82         int a= wm1010 + d40000;
83         temp[0]=a;
84 }
85
86 static void inline idctCol (int16_t * col, int16_t *input)
87 {
88 #undef C0
89 #undef C1
90 #undef C2
91 #undef C3
92 #undef C4
93 #undef C5
94 #undef C6
95 #undef C7
96         int a0, a1, a2, a3, b0, b1, b2, b3;
97         const int C0 = 23170; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
98         const int C1 = 22725; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
99         const int C2 = 21407; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
100         const int C3 = 19266; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
101         const int C4 = 16383; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
102         const int C5 = 12873; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
103         const int C6 = 8867;  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
104         const int C7 = 4520;  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
105 /*
106         if( !(col[8*1] | col[8*2] |col[8*3] |col[8*4] |col[8*5] |col[8*6] | col[8*7])) {
107                 col[8*0] = col[8*1] = col[8*2] = col[8*3] = col[8*4] =
108                         col[8*5] = col[8*6] = col[8*7] = col[8*0]<<3;
109                 return;
110         }*/
111
112 col[8*0] = input[8*0 + 0];
113 col[8*1] = input[8*2 + 0];
114 col[8*2] = input[8*0 + 1];
115 col[8*3] = input[8*2 + 1];
116 col[8*4] = input[8*4 + 0];
117 col[8*5] = input[8*6 + 0];
118 col[8*6] = input[8*4 + 1];
119 col[8*7] = input[8*6 + 1];
120
121         a0 = C4*col[8*0] + C2*col[8*2] + C4*col[8*4] + C6*col[8*6] + (1<<(COL_SHIFT-1));
122         a1 = C4*col[8*0] + C6*col[8*2] - C4*col[8*4] - C2*col[8*6] + (1<<(COL_SHIFT-1));
123         a2 = C4*col[8*0] - C6*col[8*2] - C4*col[8*4] + C2*col[8*6] + (1<<(COL_SHIFT-1));
124         a3 = C4*col[8*0] - C2*col[8*2] + C4*col[8*4] - C6*col[8*6] + (1<<(COL_SHIFT-1));
125
126         b0 = C1*col[8*1] + C3*col[8*3] + C5*col[8*5] + C7*col[8*7];
127         b1 = C3*col[8*1] - C7*col[8*3] - C1*col[8*5] - C5*col[8*7];
128         b2 = C5*col[8*1] - C1*col[8*3] + C7*col[8*5] + C3*col[8*7];
129         b3 = C7*col[8*1] - C5*col[8*3] + C3*col[8*5] - C1*col[8*7];
130
131         col[8*0] = (a0 + b0) >> COL_SHIFT;
132         col[8*1] = (a1 + b1) >> COL_SHIFT;
133         col[8*2] = (a2 + b2) >> COL_SHIFT;
134         col[8*3] = (a3 + b3) >> COL_SHIFT;
135         col[8*4] = (a3 - b3) >> COL_SHIFT;
136         col[8*5] = (a2 - b2) >> COL_SHIFT;
137         col[8*6] = (a1 - b1) >> COL_SHIFT;
138         col[8*7] = (a0 - b0) >> COL_SHIFT;
139 }
140
141 static void inline idctRow (int16_t * output, int16_t * input)
142 {
143         int16_t row[8];
144
145         int a0, a1, a2, a3, b0, b1, b2, b3;
146         const int C0 = 23170; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
147         const int C1 = 22725; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
148         const int C2 = 21407; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
149         const int C3 = 19266; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
150         const int C4 = 16383; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
151         const int C5 = 12873; //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
152         const int C6 = 8867;  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
153         const int C7 = 4520;  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
154
155 row[0] = input[0];
156 row[2] = input[1];
157 row[4] = input[4];
158 row[6] = input[5];
159 row[1] = input[8];
160 row[3] = input[9];
161 row[5] = input[12];
162 row[7] = input[13];
163
164         if( !(row[1] | row[2] |row[3] |row[4] |row[5] |row[6] | row[7]) ) {
165                 row[0] = row[1] = row[2] = row[3] = row[4] =
166                         row[5] = row[6] = row[7] = row[0]<<3;
167         output[0]  = row[0];
168         output[2]  = row[1];
169         output[4]  = row[2];
170         output[6]  = row[3];
171         output[8]  = row[4];
172         output[10] = row[5];
173         output[12] = row[6];
174         output[14] = row[7];
175                 return;
176         }
177
178         a0 = C4*row[0] + C2*row[2] + C4*row[4] + C6*row[6] + (1<<(ROW_SHIFT-1));
179         a1 = C4*row[0] + C6*row[2] - C4*row[4] - C2*row[6] + (1<<(ROW_SHIFT-1));
180         a2 = C4*row[0] - C6*row[2] - C4*row[4] + C2*row[6] + (1<<(ROW_SHIFT-1));
181         a3 = C4*row[0] - C2*row[2] + C4*row[4] - C6*row[6] + (1<<(ROW_SHIFT-1));
182
183         b0 = C1*row[1] + C3*row[3] + C5*row[5] + C7*row[7];
184         b1 = C3*row[1] - C7*row[3] - C1*row[5] - C5*row[7];
185         b2 = C5*row[1] - C1*row[3] + C7*row[5] + C3*row[7];
186         b3 = C7*row[1] - C5*row[3] + C3*row[5] - C1*row[7];
187
188         row[0] = (a0 + b0) >> ROW_SHIFT;
189         row[1] = (a1 + b1) >> ROW_SHIFT;
190         row[2] = (a2 + b2) >> ROW_SHIFT;
191         row[3] = (a3 + b3) >> ROW_SHIFT;
192         row[4] = (a3 - b3) >> ROW_SHIFT;
193         row[5] = (a2 - b2) >> ROW_SHIFT;
194         row[6] = (a1 - b1) >> ROW_SHIFT;
195         row[7] = (a0 - b0) >> ROW_SHIFT;
196
197         output[0]  = row[0];
198         output[2]  = row[1];
199         output[4]  = row[2];
200         output[6]  = row[3];
201         output[8]  = row[4];
202         output[10] = row[5];
203         output[12] = row[6];
204         output[14] = row[7];
205 }
206 #endif
207
208 static inline void idct(int16_t *block)
209 {
210         int64_t __attribute__((aligned(8))) align_tmp[16];
211         int16_t * const temp= (int16_t*)align_tmp;
212
213         asm volatile(
214 #if 0 //Alternative, simpler variant
215
216 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
217         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
218         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
219         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
220         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
221         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
222         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
223         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
224         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
225         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
226         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
227         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
228         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
229         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
230         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
231         #rounder ", %%mm4               \n\t"\
232         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
233         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
234         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
235         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
236         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
237         #rounder ", %%mm0               \n\t"\
238         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
239         "paddd %%mm0, %%mm0             \n\t" \
240         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
241         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
242         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
243         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
244         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
245         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
246         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
247         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
248         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
249         "psrad $" #shift ", %%mm7       \n\t"\
250         "psrad $" #shift ", %%mm4       \n\t"\
251         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
252         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
253         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
254         "psrad $" #shift ", %%mm1       \n\t"\
255         "psrad $" #shift ", %%mm2       \n\t"\
256         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
257         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
258         "movq %%mm7, " #dst "           \n\t"\
259         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
260         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
261         "movq %%mm2, 24+" #dst "        \n\t"\
262         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
263         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
264         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
265         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
266         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
267         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
268         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
269         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
270         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
271         "psrad $" #shift ", %%mm2       \n\t"\
272         "psrad $" #shift ", %%mm0       \n\t"\
273         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
274         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
275         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
276         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
277         "psrad $" #shift ", %%mm6       \n\t"\
278         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
279         "movq %%mm2, 8+" #dst "         \n\t"\
280         "psrad $" #shift ", %%mm4       \n\t"\
281         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
282         "movq %%mm4, 16+" #dst "        \n\t"\
283
284 #define COL_IDCT(src0, src4, src1, src5, dst, shift) \
285         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
286         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
287         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
288         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
289         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
290         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
291         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
292         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
293         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
294         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
295         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
296         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
297         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
298         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
299         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
300         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
301         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
302         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
303         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
304         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
305         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
306         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
307         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
308         "paddd %%mm1, %%mm7             \n\t" /* B0             b0 */\
309         "movq 72(%2), %%mm1             \n\t" /* -C5    -C1     -C5     -C1 */\
310         "pmaddwd %%mm3, %%mm1           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
311         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
312         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
313         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
314         "paddd %%mm2, %%mm1             \n\t" /* B1             b1 */\
315         "psrad $" #shift ", %%mm7       \n\t"\
316         "psrad $" #shift ", %%mm4       \n\t"\
317         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
318         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
319         "psubd %%mm1, %%mm2             \n\t" /* A1-B1          a1-b1 */\
320         "psrad $" #shift ", %%mm0       \n\t"\
321         "psrad $" #shift ", %%mm2       \n\t"\
322         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
323         "movd %%mm7, " #dst "           \n\t"\
324         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
325         "movd %%mm0, 16+" #dst "        \n\t"\
326         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
327         "movd %%mm2, 96+" #dst "        \n\t"\
328         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
329         "movd %%mm4, 112+" #dst "       \n\t"\
330         "movq " #src1 ", %%mm0          \n\t" /* R3     R1      r3      r1 */\
331         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
332         "pmaddwd %%mm0, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
333         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
334         "pmaddwd 96(%2), %%mm0          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
335         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
336         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
337         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
338         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
339         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
340         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
341         "psrad $" #shift ", %%mm2       \n\t"\
342         "psrad $" #shift ", %%mm5       \n\t"\
343         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
344         "paddd %%mm0, %%mm3             \n\t" /* B3             b3 */\
345         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
346         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
347         "psrad $" #shift ", %%mm6       \n\t"\
348         "psrad $" #shift ", %%mm4       \n\t"\
349         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
350         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
351         "movd %%mm2, 32+" #dst "        \n\t"\
352         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
353         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
354         "movd %%mm6, 48+" #dst "        \n\t"\
355         "movd %%mm4, 64+" #dst "        \n\t"\
356         "movd %%mm5, 80+" #dst "        \n\t"\
357
358
359 #define DC_COND_ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
360         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
361         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
362         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
363         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
364         "movq "MANGLE(wm1010)", %%mm4   \n\t"\
365         "pand %%mm0, %%mm4              \n\t"\
366         "por %%mm1, %%mm4               \n\t"\
367         "por %%mm2, %%mm4               \n\t"\
368         "por %%mm3, %%mm4               \n\t"\
369         "packssdw %%mm4,%%mm4           \n\t"\
370         "movd %%mm4, %%eax              \n\t"\
371         "orl %%eax, %%eax               \n\t"\
372         "jz 1f                          \n\t"\
373         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
374         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
375         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
376         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
377         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
378         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
379         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
380         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
381         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
382         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
383         #rounder ", %%mm4               \n\t"\
384         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
385         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
386         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
387         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
388         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
389         #rounder ", %%mm0               \n\t"\
390         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
391         "paddd %%mm0, %%mm0             \n\t" \
392         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
393         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
394         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
395         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
396         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
397         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
398         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
399         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
400         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
401         "psrad $" #shift ", %%mm7       \n\t"\
402         "psrad $" #shift ", %%mm4       \n\t"\
403         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
404         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
405         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
406         "psrad $" #shift ", %%mm1       \n\t"\
407         "psrad $" #shift ", %%mm2       \n\t"\
408         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
409         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
410         "movq %%mm7, " #dst "           \n\t"\
411         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
412         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
413         "movq %%mm2, 24+" #dst "        \n\t"\
414         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
415         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
416         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
417         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
418         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
419         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
420         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
421         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
422         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
423         "psrad $" #shift ", %%mm2       \n\t"\
424         "psrad $" #shift ", %%mm0       \n\t"\
425         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
426         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
427         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
428         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
429         "psrad $" #shift ", %%mm6       \n\t"\
430         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
431         "movq %%mm2, 8+" #dst "         \n\t"\
432         "psrad $" #shift ", %%mm4       \n\t"\
433         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
434         "movq %%mm4, 16+" #dst "        \n\t"\
435         "jmp 2f                         \n\t"\
436         "1:                             \n\t"\
437         "pslld $16, %%mm0               \n\t"\
438         "#paddd "MANGLE(d40000)", %%mm0 \n\t"\
439         "psrad $13, %%mm0               \n\t"\
440         "packssdw %%mm0, %%mm0          \n\t"\
441         "movq %%mm0, " #dst "           \n\t"\
442         "movq %%mm0, 8+" #dst "         \n\t"\
443         "movq %%mm0, 16+" #dst "        \n\t"\
444         "movq %%mm0, 24+" #dst "        \n\t"\
445         "2:                             \n\t"
446
447
448 //IDCT(      src0,   src4,   src1,   src5,    dst,    rounder, shift)
449 ROW_IDCT(    (%0),  8(%0), 16(%0), 24(%0),  0(%1),paddd 8(%2), 11)
450 /*ROW_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1), paddd (%2), 11)
451 ROW_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1), paddd (%2), 11)
452 ROW_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1), paddd (%2), 11)*/
453
454 DC_COND_ROW_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11)
455 DC_COND_ROW_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11)
456 DC_COND_ROW_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11)
457
458
459 //IDCT(      src0,   src4,   src1,    src5,    dst, shift)
460 COL_IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
461 COL_IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
462 COL_IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
463 COL_IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
464
465 #else
466
467 #define DC_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
468         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
469         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
470         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
471         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
472         "movq "MANGLE(wm1010)", %%mm4   \n\t"\
473         "pand %%mm0, %%mm4              \n\t"\
474         "por %%mm1, %%mm4               \n\t"\
475         "por %%mm2, %%mm4               \n\t"\
476         "por %%mm3, %%mm4               \n\t"\
477         "packssdw %%mm4,%%mm4           \n\t"\
478         "movd %%mm4, %%eax              \n\t"\
479         "orl %%eax, %%eax               \n\t"\
480         "jz 1f                          \n\t"\
481         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
482         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
483         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
484         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
485         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
486         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
487         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
488         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
489         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
490         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
491         #rounder ", %%mm4               \n\t"\
492         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
493         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
494         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
495         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
496         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
497         #rounder ", %%mm0               \n\t"\
498         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
499         "paddd %%mm0, %%mm0             \n\t" \
500         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
501         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
502         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
503         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
504         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
505         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
506         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
507         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
508         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
509         "psrad $" #shift ", %%mm7       \n\t"\
510         "psrad $" #shift ", %%mm4       \n\t"\
511         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
512         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
513         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
514         "psrad $" #shift ", %%mm1       \n\t"\
515         "psrad $" #shift ", %%mm2       \n\t"\
516         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
517         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
518         "movq %%mm7, " #dst "           \n\t"\
519         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
520         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
521         "movq %%mm2, 24+" #dst "        \n\t"\
522         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
523         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
524         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
525         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
526         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
527         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
528         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
529         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
530         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
531         "psrad $" #shift ", %%mm2       \n\t"\
532         "psrad $" #shift ", %%mm0       \n\t"\
533         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
534         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
535         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
536         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
537         "psrad $" #shift ", %%mm6       \n\t"\
538         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
539         "movq %%mm2, 8+" #dst "         \n\t"\
540         "psrad $" #shift ", %%mm4       \n\t"\
541         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
542         "movq %%mm4, 16+" #dst "        \n\t"\
543         "jmp 2f                         \n\t"\
544         "1:                             \n\t"\
545         "pslld $16, %%mm0               \n\t"\
546         "paddd "MANGLE(d40000)", %%mm0  \n\t"\
547         "psrad $13, %%mm0               \n\t"\
548         "packssdw %%mm0, %%mm0          \n\t"\
549         "movq %%mm0, " #dst "           \n\t"\
550         "movq %%mm0, 8+" #dst "         \n\t"\
551         "movq %%mm0, 16+" #dst "        \n\t"\
552         "movq %%mm0, 24+" #dst "        \n\t"\
553         "2:                             \n\t"
554
555 #define Z_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift, bt) \
556         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
557         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
558         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
559         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
560         "movq %%mm0, %%mm4              \n\t"\
561         "por %%mm1, %%mm4               \n\t"\
562         "por %%mm2, %%mm4               \n\t"\
563         "por %%mm3, %%mm4               \n\t"\
564         "packssdw %%mm4,%%mm4           \n\t"\
565         "movd %%mm4, %%eax              \n\t"\
566         "orl %%eax, %%eax               \n\t"\
567         "jz " #bt "                     \n\t"\
568         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
569         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
570         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
571         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
572         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
573         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
574         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
575         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
576         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
577         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
578         #rounder ", %%mm4               \n\t"\
579         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
580         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
581         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
582         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
583         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
584         #rounder ", %%mm0               \n\t"\
585         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
586         "paddd %%mm0, %%mm0             \n\t" \
587         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
588         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
589         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
590         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
591         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
592         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
593         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
594         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
595         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
596         "psrad $" #shift ", %%mm7       \n\t"\
597         "psrad $" #shift ", %%mm4       \n\t"\
598         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
599         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
600         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
601         "psrad $" #shift ", %%mm1       \n\t"\
602         "psrad $" #shift ", %%mm2       \n\t"\
603         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
604         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
605         "movq %%mm7, " #dst "           \n\t"\
606         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
607         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
608         "movq %%mm2, 24+" #dst "        \n\t"\
609         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
610         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
611         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
612         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
613         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
614         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
615         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
616         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
617         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
618         "psrad $" #shift ", %%mm2       \n\t"\
619         "psrad $" #shift ", %%mm0       \n\t"\
620         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
621         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
622         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
623         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
624         "psrad $" #shift ", %%mm6       \n\t"\
625         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
626         "movq %%mm2, 8+" #dst "         \n\t"\
627         "psrad $" #shift ", %%mm4       \n\t"\
628         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
629         "movq %%mm4, 16+" #dst "        \n\t"\
630
631 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
632         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
633         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
634         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
635         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
636         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
637         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
638         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
639         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
640         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
641         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
642         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
643         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
644         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
645         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
646         #rounder ", %%mm4               \n\t"\
647         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
648         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
649         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
650         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
651         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
652         #rounder ", %%mm0               \n\t"\
653         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
654         "paddd %%mm0, %%mm0             \n\t" \
655         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
656         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
657         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
658         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
659         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
660         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
661         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
662         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
663         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
664         "psrad $" #shift ", %%mm7       \n\t"\
665         "psrad $" #shift ", %%mm4       \n\t"\
666         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
667         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
668         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
669         "psrad $" #shift ", %%mm1       \n\t"\
670         "psrad $" #shift ", %%mm2       \n\t"\
671         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
672         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
673         "movq %%mm7, " #dst "           \n\t"\
674         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
675         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
676         "movq %%mm2, 24+" #dst "        \n\t"\
677         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
678         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
679         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
680         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
681         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
682         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
683         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
684         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
685         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
686         "psrad $" #shift ", %%mm2       \n\t"\
687         "psrad $" #shift ", %%mm0       \n\t"\
688         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
689         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
690         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
691         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
692         "psrad $" #shift ", %%mm6       \n\t"\
693         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
694         "movq %%mm2, 8+" #dst "         \n\t"\
695         "psrad $" #shift ", %%mm4       \n\t"\
696         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
697         "movq %%mm4, 16+" #dst "        \n\t"\
698
699 //IDCT(         src0,   src4,   src1,   src5,    dst,   rounder, shift)
700 DC_COND_IDCT(  0(%0),  8(%0), 16(%0), 24(%0),  0(%1),paddd 8(%2), 11)
701 Z_COND_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11, 4f)
702 Z_COND_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 2f)
703 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 1f)
704
705 #undef IDCT
706 #define IDCT(src0, src4, src1, src5, dst, shift) \
707         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
708         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
709         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
710         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
711         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
712         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
713         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
714         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
715         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
716         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
717         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
718         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
719         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
720         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
721         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
722         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
723         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
724         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
725         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
726         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
727         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
728         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
729         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
730         "paddd %%mm1, %%mm7             \n\t" /* B0             b0 */\
731         "movq 72(%2), %%mm1             \n\t" /* -C5    -C1     -C5     -C1 */\
732         "pmaddwd %%mm3, %%mm1           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
733         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
734         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
735         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
736         "paddd %%mm2, %%mm1             \n\t" /* B1             b1 */\
737         "psrad $" #shift ", %%mm7       \n\t"\
738         "psrad $" #shift ", %%mm4       \n\t"\
739         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
740         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
741         "psubd %%mm1, %%mm2             \n\t" /* A1-B1          a1-b1 */\
742         "psrad $" #shift ", %%mm0       \n\t"\
743         "psrad $" #shift ", %%mm2       \n\t"\
744         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
745         "movd %%mm7, " #dst "           \n\t"\
746         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
747         "movd %%mm0, 16+" #dst "        \n\t"\
748         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
749         "movd %%mm2, 96+" #dst "        \n\t"\
750         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
751         "movd %%mm4, 112+" #dst "       \n\t"\
752         "movq " #src1 ", %%mm0          \n\t" /* R3     R1      r3      r1 */\
753         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
754         "pmaddwd %%mm0, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
755         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
756         "pmaddwd 96(%2), %%mm0          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
757         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
758         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
759         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
760         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
761         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
762         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
763         "psrad $" #shift ", %%mm2       \n\t"\
764         "psrad $" #shift ", %%mm5       \n\t"\
765         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
766         "paddd %%mm0, %%mm3             \n\t" /* B3             b3 */\
767         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
768         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
769         "psrad $" #shift ", %%mm6       \n\t"\
770         "psrad $" #shift ", %%mm4       \n\t"\
771         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
772         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
773         "movd %%mm2, 32+" #dst "        \n\t"\
774         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
775         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
776         "movd %%mm6, 48+" #dst "        \n\t"\
777         "movd %%mm4, 64+" #dst "        \n\t"\
778         "movd %%mm5, 80+" #dst "        \n\t"
779
780
781 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
782 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
783 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
784 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
785 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
786         "jmp 9f                         \n\t"
787
788         "#.balign 16                    \n\t"\
789         "4:                             \n\t"
790 Z_COND_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 6f)
791 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 5f)
792
793 #undef IDCT
794 #define IDCT(src0, src4, src1, src5, dst, shift) \
795         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
796         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
797         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
798         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
799         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
800         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
801         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
802         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
803         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
804         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
805         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
806         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
807         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
808         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
809         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
810         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
811         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
812         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
813         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
814         "movq 72(%2), %%mm7             \n\t" /* -C5    -C1     -C5     -C1 */\
815         "pmaddwd %%mm3, %%mm7           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
816         "paddd %%mm4, %%mm1             \n\t" /* A0+B0          a0+b0 */\
817         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
818         "psubd %%mm1, %%mm4             \n\t" /* A0-B0          a0-b0 */\
819         "psrad $" #shift ", %%mm1       \n\t"\
820         "psrad $" #shift ", %%mm4       \n\t"\
821         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
822         "paddd %%mm7, %%mm0             \n\t" /* A1+B1          a1+b1 */\
823         "psubd %%mm7, %%mm2             \n\t" /* A1-B1          a1-b1 */\
824         "psrad $" #shift ", %%mm0       \n\t"\
825         "psrad $" #shift ", %%mm2       \n\t"\
826         "packssdw %%mm1, %%mm1          \n\t" /* A0+B0  a0+b0 */\
827         "movd %%mm1, " #dst "           \n\t"\
828         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
829         "movd %%mm0, 16+" #dst "        \n\t"\
830         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
831         "movd %%mm2, 96+" #dst "        \n\t"\
832         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
833         "movd %%mm4, 112+" #dst "       \n\t"\
834         "movq 88(%2), %%mm1             \n\t" /* C3     C7      C3      C7 */\
835         "pmaddwd %%mm3, %%mm1           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
836         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
837         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
838         "paddd %%mm1, %%mm2             \n\t" /* A2+B2          a2+b2 */\
839         "psubd %%mm1, %%mm5             \n\t" /* a2-B2          a2-b2 */\
840         "psrad $" #shift ", %%mm2       \n\t"\
841         "psrad $" #shift ", %%mm5       \n\t"\
842         "movq %%mm6, %%mm1              \n\t" /* A3             a3 */\
843         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
844         "psubd %%mm3, %%mm1             \n\t" /* a3-B3          a3-b3 */\
845         "psrad $" #shift ", %%mm6       \n\t"\
846         "psrad $" #shift ", %%mm1       \n\t"\
847         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
848         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
849         "movd %%mm2, 32+" #dst "        \n\t"\
850         "packssdw %%mm1, %%mm1          \n\t" /* A3-B3  a3-b3 */\
851         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
852         "movd %%mm6, 48+" #dst "        \n\t"\
853         "movd %%mm1, 64+" #dst "        \n\t"\
854         "movd %%mm5, 80+" #dst "        \n\t"
855
856 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
857 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
858 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
859 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
860 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
861         "jmp 9f                         \n\t"
862
863         "#.balign 16                    \n\t"\
864         "6:                             \n\t"
865 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 7f)
866
867 #undef IDCT
868 #define IDCT(src0, src4, src1, src5, dst, shift) \
869         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
870         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
871         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
872         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
873         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
874         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
875         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
876         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
877         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
878         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
879         "movq 72(%2), %%mm7             \n\t" /* -C5    -C1     -C5     -C1 */\
880         "pmaddwd %%mm3, %%mm7           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
881         "paddd %%mm4, %%mm1             \n\t" /* A0+B0          a0+b0 */\
882         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
883         "psubd %%mm1, %%mm4             \n\t" /* A0-B0          a0-b0 */\
884         "psrad $" #shift ", %%mm1       \n\t"\
885         "psrad $" #shift ", %%mm4       \n\t"\
886         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
887         "paddd %%mm7, %%mm0             \n\t" /* A1+B1          a1+b1 */\
888         "psubd %%mm7, %%mm2             \n\t" /* A1-B1          a1-b1 */\
889         "psrad $" #shift ", %%mm0       \n\t"\
890         "psrad $" #shift ", %%mm2       \n\t"\
891         "packssdw %%mm1, %%mm1          \n\t" /* A0+B0  a0+b0 */\
892         "movd %%mm1, " #dst "           \n\t"\
893         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
894         "movd %%mm0, 16+" #dst "        \n\t"\
895         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
896         "movd %%mm2, 96+" #dst "        \n\t"\
897         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
898         "movd %%mm4, 112+" #dst "       \n\t"\
899         "movq 88(%2), %%mm1             \n\t" /* C3     C7      C3      C7 */\
900         "pmaddwd %%mm3, %%mm1           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
901         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
902         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
903         "paddd %%mm1, %%mm2             \n\t" /* A2+B2          a2+b2 */\
904         "psubd %%mm1, %%mm5             \n\t" /* a2-B2          a2-b2 */\
905         "psrad $" #shift ", %%mm2       \n\t"\
906         "psrad $" #shift ", %%mm5       \n\t"\
907         "movq %%mm6, %%mm1              \n\t" /* A3             a3 */\
908         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
909         "psubd %%mm3, %%mm1             \n\t" /* a3-B3          a3-b3 */\
910         "psrad $" #shift ", %%mm6       \n\t"\
911         "psrad $" #shift ", %%mm1       \n\t"\
912         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
913         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
914         "movd %%mm2, 32+" #dst "        \n\t"\
915         "packssdw %%mm1, %%mm1          \n\t" /* A3-B3  a3-b3 */\
916         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
917         "movd %%mm6, 48+" #dst "        \n\t"\
918         "movd %%mm1, 64+" #dst "        \n\t"\
919         "movd %%mm5, 80+" #dst "        \n\t"
920
921
922 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
923 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
924 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
925 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
926 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
927         "jmp 9f                         \n\t"
928
929         "#.balign 16                    \n\t"\
930         "2:                             \n\t"
931 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 3f)
932
933 #undef IDCT
934 #define IDCT(src0, src4, src1, src5, dst, shift) \
935         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
936         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
937         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
938         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
939         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
940         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
941         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
942         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
943         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
944         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
945         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
946         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
947         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
948         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
949         "paddd %%mm1, %%mm7             \n\t" /* B0             b0 */\
950         "movq 72(%2), %%mm1             \n\t" /* -C5    -C1     -C5     -C1 */\
951         "pmaddwd %%mm3, %%mm1           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
952         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
953         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
954         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
955         "paddd %%mm2, %%mm1             \n\t" /* B1             b1 */\
956         "psrad $" #shift ", %%mm7       \n\t"\
957         "psrad $" #shift ", %%mm4       \n\t"\
958         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
959         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
960         "psubd %%mm1, %%mm2             \n\t" /* A1-B1          a1-b1 */\
961         "psrad $" #shift ", %%mm0       \n\t"\
962         "psrad $" #shift ", %%mm2       \n\t"\
963         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
964         "movd %%mm7, " #dst "           \n\t"\
965         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
966         "movd %%mm0, 16+" #dst "        \n\t"\
967         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
968         "movd %%mm2, 96+" #dst "        \n\t"\
969         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
970         "movd %%mm4, 112+" #dst "       \n\t"\
971         "movq " #src1 ", %%mm0          \n\t" /* R3     R1      r3      r1 */\
972         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
973         "pmaddwd %%mm0, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
974         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
975         "pmaddwd 96(%2), %%mm0          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
976         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
977         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
978         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
979         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
980         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
981         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
982         "psrad $" #shift ", %%mm2       \n\t"\
983         "psrad $" #shift ", %%mm5       \n\t"\
984         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
985         "paddd %%mm0, %%mm3             \n\t" /* B3             b3 */\
986         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
987         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
988         "psrad $" #shift ", %%mm6       \n\t"\
989         "psrad $" #shift ", %%mm4       \n\t"\
990         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
991         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
992         "movd %%mm2, 32+" #dst "        \n\t"\
993         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
994         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
995         "movd %%mm6, 48+" #dst "        \n\t"\
996         "movd %%mm4, 64+" #dst "        \n\t"\
997         "movd %%mm5, 80+" #dst "        \n\t"
998
999 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1000 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1001 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1002 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1003 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1004         "jmp 9f                         \n\t"
1005
1006         "#.balign 16                    \n\t"\
1007         "3:                             \n\t"
1008 #undef IDCT
1009 #define IDCT(src0, src4, src1, src5, dst, shift) \
1010         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
1011         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
1012         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
1013         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1014         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
1015         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1016         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1017         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
1018         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
1019         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1020         "movq 64(%2), %%mm3             \n\t"\
1021         "pmaddwd %%mm2, %%mm3           \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
1022         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
1023         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
1024         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
1025         "psrad $" #shift ", %%mm7       \n\t"\
1026         "psrad $" #shift ", %%mm4       \n\t"\
1027         "movq %%mm0, %%mm1              \n\t" /* A1             a1 */\
1028         "paddd %%mm3, %%mm0             \n\t" /* A1+B1          a1+b1 */\
1029         "psubd %%mm3, %%mm1             \n\t" /* A1-B1          a1-b1 */\
1030         "psrad $" #shift ", %%mm0       \n\t"\
1031         "psrad $" #shift ", %%mm1       \n\t"\
1032         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
1033         "movd %%mm7, " #dst "           \n\t"\
1034         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
1035         "movd %%mm0, 16+" #dst "        \n\t"\
1036         "packssdw %%mm1, %%mm1          \n\t" /* A1-B1  a1-b1 */\
1037         "movd %%mm1, 96+" #dst "        \n\t"\
1038         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
1039         "movd %%mm4, 112+" #dst "       \n\t"\
1040         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
1041         "pmaddwd %%mm2, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
1042         "pmaddwd 96(%2), %%mm2          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
1043         "movq %%mm5, %%mm1              \n\t" /* A2             a2 */\
1044         "paddd %%mm4, %%mm1             \n\t" /* A2+B2          a2+b2 */\
1045         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
1046         "psrad $" #shift ", %%mm1       \n\t"\
1047         "psrad $" #shift ", %%mm5       \n\t"\
1048         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
1049         "paddd %%mm2, %%mm6             \n\t" /* A3+B3          a3+b3 */\
1050         "psubd %%mm2, %%mm4             \n\t" /* a3-B3          a3-b3 */\
1051         "psrad $" #shift ", %%mm6       \n\t"\
1052         "psrad $" #shift ", %%mm4       \n\t"\
1053         "packssdw %%mm1, %%mm1          \n\t" /* A2+B2  a2+b2 */\
1054         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
1055         "movd %%mm1, 32+" #dst "        \n\t"\
1056         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
1057         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
1058         "movd %%mm6, 48+" #dst "        \n\t"\
1059         "movd %%mm4, 64+" #dst "        \n\t"\
1060         "movd %%mm5, 80+" #dst "        \n\t"
1061
1062
1063 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1064 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1065 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1066 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1067 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1068         "jmp 9f                         \n\t"
1069
1070         "#.balign 16                    \n\t"\
1071         "5:                             \n\t"
1072 #undef IDCT
1073 #define IDCT(src0, src4, src1, src5, dst, shift) \
1074         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
1075         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
1076         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
1077         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1078         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
1079         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1080         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
1081         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
1082         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
1083         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
1084         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1085         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
1086         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
1087         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1088         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
1089         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
1090         "movq 8+" #src0 ", %%mm2        \n\t" /* R4     R0      r4      r0 */\
1091         "movq 8+" #src4 ", %%mm3        \n\t" /* R6     R2      r6      r2 */\
1092         "movq 16(%2), %%mm1             \n\t" /* C4     C4      C4      C4 */\
1093         "pmaddwd %%mm2, %%mm1           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1094         "movq 24(%2), %%mm7             \n\t" /* -C4    C4      -C4     C4 */\
1095         "pmaddwd %%mm7, %%mm2           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1096         "movq 32(%2), %%mm7             \n\t" /* C6     C2      C6      C2 */\
1097         "pmaddwd %%mm3, %%mm7           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
1098         "pmaddwd 40(%2), %%mm3          \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
1099         "paddd %%mm1, %%mm7             \n\t" /* A0             a0 */\
1100         "paddd %%mm1, %%mm1             \n\t" /* 2C0            2c0 */\
1101         "psubd %%mm7, %%mm1             \n\t" /* A3             a3 */\
1102         "paddd %%mm2, %%mm3             \n\t" /* A1             a1 */\
1103         "paddd %%mm2, %%mm2             \n\t" /* 2C1            2c1 */\
1104         "psubd %%mm3, %%mm2             \n\t" /* A2             a2 */\
1105         "psrad $" #shift ", %%mm4       \n\t"\
1106         "psrad $" #shift ", %%mm7       \n\t"\
1107         "psrad $" #shift ", %%mm3       \n\t"\
1108         "packssdw %%mm7, %%mm4          \n\t" /* A0     a0 */\
1109         "movq %%mm4, " #dst "           \n\t"\
1110         "psrad $" #shift ", %%mm0       \n\t"\
1111         "packssdw %%mm3, %%mm0          \n\t" /* A1     a1 */\
1112         "movq %%mm0, 16+" #dst "        \n\t"\
1113         "movq %%mm0, 96+" #dst "        \n\t"\
1114         "movq %%mm4, 112+" #dst "       \n\t"\
1115         "psrad $" #shift ", %%mm5       \n\t"\
1116         "psrad $" #shift ", %%mm6       \n\t"\
1117         "psrad $" #shift ", %%mm2       \n\t"\
1118         "packssdw %%mm2, %%mm5          \n\t" /* A2-B2  a2-b2 */\
1119         "movq %%mm5, 32+" #dst "        \n\t"\
1120         "psrad $" #shift ", %%mm1       \n\t"\
1121         "packssdw %%mm1, %%mm6          \n\t" /* A3+B3  a3+b3 */\
1122         "movq %%mm6, 48+" #dst "        \n\t"\
1123         "movq %%mm6, 64+" #dst "        \n\t"\
1124         "movq %%mm5, 80+" #dst "        \n\t"
1125
1126
1127 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1128 IDCT(    0(%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1129 //IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1130 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1131 //IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1132         "jmp 9f                         \n\t"
1133
1134
1135         "#.balign 16                    \n\t"\
1136         "1:                             \n\t"
1137 #undef IDCT
1138 #define IDCT(src0, src4, src1, src5, dst, shift) \
1139         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
1140         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
1141         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
1142         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
1143         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1144         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
1145         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1146         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
1147         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
1148         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
1149         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
1150         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1151         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
1152         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
1153         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
1154         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
1155         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1156         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
1157         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
1158         "movq 64(%2), %%mm1             \n\t"\
1159         "pmaddwd %%mm2, %%mm1           \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
1160         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
1161         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
1162         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
1163         "psrad $" #shift ", %%mm7       \n\t"\
1164         "psrad $" #shift ", %%mm4       \n\t"\
1165         "movq %%mm0, %%mm3              \n\t" /* A1             a1 */\
1166         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
1167         "psubd %%mm1, %%mm3             \n\t" /* A1-B1          a1-b1 */\
1168         "psrad $" #shift ", %%mm0       \n\t"\
1169         "psrad $" #shift ", %%mm3       \n\t"\
1170         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
1171         "movd %%mm7, " #dst "           \n\t"\
1172         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
1173         "movd %%mm0, 16+" #dst "        \n\t"\
1174         "packssdw %%mm3, %%mm3          \n\t" /* A1-B1  a1-b1 */\
1175         "movd %%mm3, 96+" #dst "        \n\t"\
1176         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
1177         "movd %%mm4, 112+" #dst "       \n\t"\
1178         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
1179         "pmaddwd %%mm2, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
1180         "pmaddwd 96(%2), %%mm2          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
1181         "movq %%mm5, %%mm3              \n\t" /* A2             a2 */\
1182         "paddd %%mm4, %%mm3             \n\t" /* A2+B2          a2+b2 */\
1183         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
1184         "psrad $" #shift ", %%mm3       \n\t"\
1185         "psrad $" #shift ", %%mm5       \n\t"\
1186         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
1187         "paddd %%mm2, %%mm6             \n\t" /* A3+B3          a3+b3 */\
1188         "psubd %%mm2, %%mm4             \n\t" /* a3-B3          a3-b3 */\
1189         "psrad $" #shift ", %%mm6       \n\t"\
1190         "packssdw %%mm3, %%mm3          \n\t" /* A2+B2  a2+b2 */\
1191         "movd %%mm3, 32+" #dst "        \n\t"\
1192         "psrad $" #shift ", %%mm4       \n\t"\
1193         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
1194         "movd %%mm6, 48+" #dst "        \n\t"\
1195         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
1196         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
1197         "movd %%mm4, 64+" #dst "        \n\t"\
1198         "movd %%mm5, 80+" #dst "        \n\t"
1199
1200
1201 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1202 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1203 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1204 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1205 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1206         "jmp 9f                         \n\t"
1207
1208
1209         "#.balign 16                    \n\t"
1210         "7:                             \n\t"
1211 #undef IDCT
1212 #define IDCT(src0, src4, src1, src5, dst, shift) \
1213         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
1214         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
1215         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1216         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
1217         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1218         "psrad $" #shift ", %%mm4       \n\t"\
1219         "psrad $" #shift ", %%mm0       \n\t"\
1220         "movq 8+" #src0 ", %%mm2        \n\t" /* R4     R0      r4      r0 */\
1221         "movq 16(%2), %%mm1             \n\t" /* C4     C4      C4      C4 */\
1222         "pmaddwd %%mm2, %%mm1           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
1223         "movq 24(%2), %%mm7             \n\t" /* -C4    C4      -C4     C4 */\
1224         "pmaddwd %%mm7, %%mm2           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
1225         "movq 32(%2), %%mm7             \n\t" /* C6     C2      C6      C2 */\
1226         "psrad $" #shift ", %%mm1       \n\t"\
1227         "packssdw %%mm1, %%mm4          \n\t" /* A0     a0 */\
1228         "movq %%mm4, " #dst "           \n\t"\
1229         "psrad $" #shift ", %%mm2       \n\t"\
1230         "packssdw %%mm2, %%mm0          \n\t" /* A1     a1 */\
1231         "movq %%mm0, 16+" #dst "        \n\t"\
1232         "movq %%mm0, 96+" #dst "        \n\t"\
1233         "movq %%mm4, 112+" #dst "       \n\t"\
1234         "movq %%mm0, 32+" #dst "        \n\t"\
1235         "movq %%mm4, 48+" #dst "        \n\t"\
1236         "movq %%mm4, 64+" #dst "        \n\t"\
1237         "movq %%mm0, 80+" #dst "        \n\t"
1238
1239 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
1240 IDCT(   0(%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
1241 //IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
1242 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
1243 //IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1244
1245
1246 #endif
1247
1248 /*
1249 Input
1250  00 40 04 44 20 60 24 64
1251  10 30 14 34 50 70 54 74
1252  01 41 03 43 21 61 23 63
1253  11 31 13 33 51 71 53 73
1254  02 42 06 46 22 62 26 66
1255  12 32 16 36 52 72 56 76
1256  05 45 07 47 25 65 27 67
1257  15 35 17 37 55 75 57 77
1258
1259 Temp
1260  00 04 10 14 20 24 30 34
1261  40 44 50 54 60 64 70 74
1262  01 03 11 13 21 23 31 33
1263  41 43 51 53 61 63 71 73
1264  02 06 12 16 22 26 32 36
1265  42 46 52 56 62 66 72 76
1266  05 07 15 17 25 27 35 37
1267  45 47 55 57 65 67 75 77
1268 */
1269
1270 "9: \n\t"
1271                 :: "r" (block), "r" (temp), "r" (coeffs)
1272                 : "%eax"
1273         );
1274 }
1275
1276 void ff_simple_idct_mmx(int16_t *block)
1277 {
1278     idct(block);
1279 }
1280
1281 //FIXME merge add/put into the idct
1282
1283 void ff_simple_idct_put_mmx(uint8_t *dest, int line_size, DCTELEM *block)
1284 {
1285     idct(block);
1286     put_pixels_clamped_mmx(block, dest, line_size);
1287 }
1288 void ff_simple_idct_add_mmx(uint8_t *dest, int line_size, DCTELEM *block)
1289 {
1290     idct(block);
1291     add_pixels_clamped_mmx(block, dest, line_size);
1292 }