Merge commit 'a6a750c7ef240b72ce01e9653343a0ddf247d196'
[ffmpeg.git] / libavcodec / mips / h264pred_mmi.c
1 /*
2  * Loongson SIMD optimized h264pred
3  *
4  * Copyright (c) 2015 Loongson Technology Corporation Limited
5  * Copyright (c) 2015 Zhou Xiaoyong <zhouxiaoyong@loongson.cn>
6  *                    Zhang Shuangshuang <zhangshuangshuang@ict.ac.cn>
7  *
8  * This file is part of FFmpeg.
9  *
10  * FFmpeg is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * FFmpeg is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with FFmpeg; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24
25 #include "h264pred_mips.h"
26 #include "libavcodec/bit_depth_template.c"
27 #include "libavutil/mips/asmdefs.h"
28 #include "constants.h"
29
30 void ff_pred16x16_vertical_8_mmi(uint8_t *src, ptrdiff_t stride)
31 {
32     double ftmp[2];
33     uint64_t tmp[1];
34
35     __asm__ volatile (
36         "dli        %[tmp0],    0x08                                    \n\t"
37         "gsldlc1    %[ftmp0],   0x07(%[srcA])                           \n\t"
38         "gsldrc1    %[ftmp0],   0x00(%[srcA])                           \n\t"
39         "gsldlc1    %[ftmp1],   0x0f(%[srcA])                           \n\t"
40         "gsldrc1    %[ftmp1],   0x08(%[srcA])                           \n\t"
41         "1:                                                             \n\t"
42         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
43         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
44         "gssdlc1    %[ftmp1],   0x0f(%[src])                            \n\t"
45         "gssdrc1    %[ftmp1],   0x08(%[src])                            \n\t"
46         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
47         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
48         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
49         "gssdlc1    %[ftmp1],   0x0f(%[src])                            \n\t"
50         "gssdrc1    %[ftmp1],   0x08(%[src])                            \n\t"
51         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
52         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
53         "bnez       %[tmp0],    1b                                      \n\t"
54         : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
55           [tmp0]"=&r"(tmp[0]),
56           [src]"+&r"(src)
57         : [stride]"r"((mips_reg)stride),    [srcA]"r"((mips_reg)(src-stride))
58         : "memory"
59     );
60 }
61
62 void ff_pred16x16_horizontal_8_mmi(uint8_t *src, ptrdiff_t stride)
63 {
64     uint64_t tmp[3];
65     mips_reg addr[2];
66
67     __asm__ volatile (
68         PTR_ADDI   "%[addr0],   %[src],         -0x01                   \n\t"
69         PTR_ADDU   "%[addr1],   %[src],         $0                      \n\t"
70         "dli        %[tmp2],    0x08                                    \n\t"
71         "1:                                                             \n\t"
72         "lbu        %[tmp0],    0x00(%[addr0])                          \n\t"
73         "dmul       %[tmp1],    %[tmp0],        %[ff_pb_1]              \n\t"
74         "swl        %[tmp1],    0x07(%[addr1])                          \n\t"
75         "swr        %[tmp1],    0x00(%[addr1])                          \n\t"
76         "swl        %[tmp1],    0x0f(%[addr1])                          \n\t"
77         "swr        %[tmp1],    0x08(%[addr1])                          \n\t"
78         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
79         PTR_ADDU   "%[addr1],   %[addr1],       %[stride]               \n\t"
80         "lbu        %[tmp0],    0x00(%[addr0])                          \n\t"
81         "dmul       %[tmp1],    %[tmp0],        %[ff_pb_1]              \n\t"
82         "swl        %[tmp1],    0x07(%[addr1])                          \n\t"
83         "swr        %[tmp1],    0x00(%[addr1])                          \n\t"
84         "swl        %[tmp1],    0x0f(%[addr1])                          \n\t"
85         "swr        %[tmp1],    0x08(%[addr1])                          \n\t"
86         "daddi      %[tmp2],    %[tmp2],        -0x01                   \n\t"
87         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
88         PTR_ADDU   "%[addr1],   %[addr1],       %[stride]               \n\t"
89         "bnez       %[tmp2],    1b                                      \n\t"
90         : [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
91           [tmp2]"=&r"(tmp[2]),
92           [addr0]"=&r"(addr[0]),            [addr1]"=&r"(addr[1])
93         : [src]"r"((mips_reg)src),          [stride]"r"((mips_reg)stride),
94           [ff_pb_1]"r"(ff_pb_1)
95         : "memory"
96     );
97 }
98
99 void ff_pred16x16_dc_8_mmi(uint8_t *src, ptrdiff_t stride)
100 {
101     uint64_t tmp[4];
102     mips_reg addr[2];
103
104     __asm__ volatile (
105         PTR_ADDI   "%[addr0],   %[src],         -0x01                   \n\t"
106         "dli        %[tmp0],    0x08                                    \n\t"
107         "xor        %[tmp3],    %[tmp3],        %[tmp3]                 \n\t"
108         "1:                                                             \n\t"
109         "lbu        %[tmp1],    0x00(%[addr0])                          \n\t"
110         "daddu      %[tmp3],    %[tmp3],        %[tmp1]                 \n\t"
111         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
112         "lbu        %[tmp1],    0x00(%[addr0])                          \n\t"
113         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
114         "daddu      %[tmp3],    %[tmp3],        %[tmp1]                 \n\t"
115         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
116         "bnez       %[tmp0],    1b                                      \n\t"
117
118         "dli        %[tmp0],    0x08                                    \n\t"
119         PTR_SUBU   "%[addr0],   %[src],         %[stride]               \n\t"
120         "2:                                                             \n\t"
121         "lbu        %[tmp1],    0x00(%[addr0])                          \n\t"
122         "daddu      %[tmp3],    %[tmp3],        %[tmp1]                 \n\t"
123         PTR_ADDIU  "%[addr0],   %[addr0],       0x01                    \n\t"
124         "lbu        %[tmp1],    0x00(%[addr0])                          \n\t"
125         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
126         "daddu      %[tmp3],    %[tmp3],        %[tmp1]                 \n\t"
127         PTR_ADDIU  "%[addr0],   %[addr0],       0x01                    \n\t"
128         "bnez       %[tmp0],    2b                                      \n\t"
129
130         "daddiu     %[tmp3],    %[tmp3],        0x10                    \n\t"
131         "dsra       %[tmp3],    0x05                                    \n\t"
132         "dmul       %[tmp2],    %[tmp3],        %[ff_pb_1]              \n\t"
133         PTR_ADDU   "%[addr0],   %[src],         $0                      \n\t"
134         "dli        %[tmp0],    0x08                                    \n\t"
135         "3:                                                             \n\t"
136         "swl        %[tmp2],    0x07(%[addr0])                          \n\t"
137         "swr        %[tmp2],    0x00(%[addr0])                          \n\t"
138         "swl        %[tmp2],    0x0f(%[addr0])                          \n\t"
139         "swr        %[tmp2],    0x08(%[addr0])                          \n\t"
140         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
141         "swl        %[tmp2],    0x07(%[addr0])                          \n\t"
142         "swr        %[tmp2],    0x00(%[addr0])                          \n\t"
143         "swl        %[tmp2],    0x0f(%[addr0])                          \n\t"
144         "swr        %[tmp2],    0x08(%[addr0])                          \n\t"
145         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
146         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
147         "bnez       %[tmp0],    3b                                      \n\t"
148         : [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
149           [tmp2]"=&r"(tmp[2]),              [tmp3]"=&r"(tmp[3]),
150           [addr0]"=&r"(addr[0]),            [addr1]"=&r"(addr[1])
151         : [src]"r"((mips_reg)src),          [stride]"r"((mips_reg)stride),
152           [ff_pb_1]"r"(ff_pb_1)
153         : "memory"
154     );
155 }
156
157 void ff_pred8x8l_top_dc_8_mmi(uint8_t *src, int has_topleft,
158         int has_topright, ptrdiff_t stride)
159 {
160     uint32_t dc;
161     double ftmp[11];
162     mips_reg tmp[3];
163
164     __asm__ volatile (
165         "xor        %[ftmp0],   %[ftmp0],       %[ftmp0]                \n\t"
166         "gsldlc1    %[ftmp10],  0x07(%[srcA])                           \n\t"
167         "gsldrc1    %[ftmp10],  0x00(%[srcA])                           \n\t"
168         "gsldlc1    %[ftmp9],   0x07(%[src0])                           \n\t"
169         "gsldrc1    %[ftmp9],   0x00(%[src0])                           \n\t"
170         "gsldlc1    %[ftmp8],   0x07(%[src1])                           \n\t"
171         "gsldrc1    %[ftmp8],   0x00(%[src1])                           \n\t"
172
173         "punpcklbh  %[ftmp7],   %[ftmp10],      %[ftmp0]                \n\t"
174         "punpckhbh  %[ftmp6],   %[ftmp10],      %[ftmp0]                \n\t"
175         "punpcklbh  %[ftmp5],   %[ftmp9],       %[ftmp0]                \n\t"
176         "punpckhbh  %[ftmp4],   %[ftmp9],       %[ftmp0]                \n\t"
177         "punpcklbh  %[ftmp3],   %[ftmp8],       %[ftmp0]                \n\t"
178         "punpckhbh  %[ftmp2],   %[ftmp8],       %[ftmp0]                \n\t"
179         "bnez       %[has_topleft],             1f                      \n\t"
180         "pinsrh_0   %[ftmp7],   %[ftmp7],       %[ftmp5]                \n\t"
181
182         "1:                                                             \n\t"
183         "bnez       %[has_topright],            2f                      \n\t"
184         "pinsrh_3   %[ftmp2],   %[ftmp2],       %[ftmp4]                \n\t"
185
186         "2:                                                             \n\t"
187         "dli        %[tmp0],    0x02                                    \n\t"
188         "mtc1       %[tmp0],    %[ftmp1]                                \n\t"
189         "pmullh     %[ftmp5],   %[ftmp5],       %[ff_pw_2]              \n\t"
190         "pmullh     %[ftmp4],   %[ftmp4],       %[ff_pw_2]              \n\t"
191         "paddh      %[ftmp7],   %[ftmp7],       %[ftmp5]                \n\t"
192         "paddh      %[ftmp6],   %[ftmp6],       %[ftmp4]                \n\t"
193         "paddh      %[ftmp7],   %[ftmp7],       %[ftmp3]                \n\t"
194         "paddh      %[ftmp6],   %[ftmp6],       %[ftmp2]                \n\t"
195         "paddh      %[ftmp7],   %[ftmp7],       %[ff_pw_2]              \n\t"
196         "paddh      %[ftmp6],   %[ftmp6],       %[ff_pw_2]              \n\t"
197         "psrah      %[ftmp7],   %[ftmp7],       %[ftmp1]                \n\t"
198         "psrah      %[ftmp6],   %[ftmp6],       %[ftmp1]                \n\t"
199         "packushb   %[ftmp9],   %[ftmp7],       %[ftmp6]                \n\t"
200         "biadd      %[ftmp10],  %[ftmp9]                                \n\t"
201         "mfc1       %[tmp1],    %[ftmp10]                               \n\t"
202         "addiu      %[tmp1],    %[tmp1],        0x04                    \n\t"
203         "srl        %[tmp1],    %[tmp1],        0x03                    \n\t"
204         "mul        %[dc],      %[tmp1],        %[ff_pb_1]              \n\t"
205         : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
206           [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
207           [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
208           [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
209           [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
210           [ftmp10]"=&f"(ftmp[10]),
211           [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
212           [dc]"=r"(dc)
213         : [srcA]"r"((mips_reg)(src-stride-1)),
214           [src0]"r"((mips_reg)(src-stride)),
215           [src1]"r"((mips_reg)(src-stride+1)),
216           [has_topleft]"r"(has_topleft),    [has_topright]"r"(has_topright),
217           [ff_pb_1]"r"(ff_pb_1),            [ff_pw_2]"f"(ff_pw_2)
218         : "memory"
219     );
220
221     __asm__ volatile (
222         "dli        %[tmp0],    0x02                                    \n\t"
223         "punpcklwd  %[ftmp0],   %[dc],          %[dc]                   \n\t"
224         "1:                                                             \n\t"
225         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
226         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
227         "gssdxc1    %[ftmp0],   0x00(%[src],    %[stride])              \n\t"
228         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
229         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
230         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
231         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
232         "gssdxc1    %[ftmp0],   0x00(%[src],    %[stride])              \n\t"
233         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
234         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
235         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
236         "bnez       %[tmp0],    1b                                      \n\t"
237         : [ftmp0]"=&f"(ftmp[0]),            [tmp0]"=&r"(tmp[0]),
238           [src]"+&r"(src)
239         : [dc]"f"(dc),                      [stride]"r"((mips_reg)stride)
240         : "memory"
241     );
242 }
243
244 void ff_pred8x8l_dc_8_mmi(uint8_t *src, int has_topleft, int has_topright,
245         ptrdiff_t stride)
246 {
247     uint32_t dc, dc1, dc2;
248     double ftmp[14];
249     mips_reg tmp[1];
250
251     const int l0 = ((has_topleft ? src[-1+-1*stride] : src[-1+0*stride]) + 2*src[-1+0*stride] + src[-1+1*stride] + 2) >> 2;
252     const int l1 = (src[-1+0*stride] + 2*src[-1+1*stride] + src[-1+2*stride] + 2) >> 2;
253     const int l2 = (src[-1+1*stride] + 2*src[-1+2*stride] + src[-1+3*stride] + 2) >> 2;
254     const int l3 = (src[-1+2*stride] + 2*src[-1+3*stride] + src[-1+4*stride] + 2) >> 2;
255     const int l4 = (src[-1+3*stride] + 2*src[-1+4*stride] + src[-1+5*stride] + 2) >> 2;
256     const int l5 = (src[-1+4*stride] + 2*src[-1+5*stride] + src[-1+6*stride] + 2) >> 2;
257     const int l6 = (src[-1+5*stride] + 2*src[-1+6*stride] + src[-1+7*stride] + 2) >> 2;
258     const int l7 = (src[-1+6*stride] + 2*src[-1+7*stride] + src[-1+7*stride] + 2) >> 2;
259
260     __asm__ volatile (
261         "gsldlc1    %[ftmp4],   0x07(%[srcA])                           \n\t"
262         "gsldrc1    %[ftmp4],   0x00(%[srcA])                           \n\t"
263         "gsldlc1    %[ftmp5],   0x07(%[src0])                           \n\t"
264         "gsldrc1    %[ftmp5],   0x00(%[src0])                           \n\t"
265         "gsldlc1    %[ftmp6],   0x07(%[src1])                           \n\t"
266         "gsldrc1    %[ftmp6],   0x00(%[src1])                           \n\t"
267         "xor        %[ftmp0],   %[ftmp0],       %[ftmp0]                \n\t"
268         "dli        %[tmp0],    0x03                                    \n\t"
269         "punpcklbh  %[ftmp7],   %[ftmp4],       %[ftmp0]                \n\t"
270         "punpckhbh  %[ftmp8],   %[ftmp4],       %[ftmp0]                \n\t"
271         "mtc1       %[tmp0],    %[ftmp1]                                \n\t"
272         "punpcklbh  %[ftmp9],   %[ftmp5],       %[ftmp0]                \n\t"
273         "punpckhbh  %[ftmp10],  %[ftmp5],       %[ftmp0]                \n\t"
274         "punpcklbh  %[ftmp11],  %[ftmp6],       %[ftmp0]                \n\t"
275         "punpckhbh  %[ftmp12],  %[ftmp6],       %[ftmp0]                \n\t"
276         "pshufh     %[ftmp3],   %[ftmp8],       %[ftmp1]                \n\t"
277         "pshufh     %[ftmp13],  %[ftmp12],      %[ftmp1]                \n\t"
278         "pinsrh_3   %[ftmp8],   %[ftmp8],       %[ftmp13]               \n\t"
279         "pinsrh_3   %[ftmp12],  %[ftmp12],      %[ftmp3]                \n\t"
280         "bnez       %[has_topleft],             1f                      \n\t"
281         "pinsrh_0   %[ftmp7],   %[ftmp7],       %[ftmp9]                \n\t"
282
283         "1:                                                             \n\t"
284         "bnez       %[has_topright],            2f                      \n\t"
285         "pshufh     %[ftmp13],  %[ftmp10],      %[ftmp1]                \n\t"
286         "pinsrh_3   %[ftmp8],   %[ftmp8],       %[ftmp13]               \n\t"
287
288         "2:                                                             \n\t"
289         "dli        %[tmp0],    0x02                                    \n\t"
290         "mtc1       %[tmp0],    %[ftmp1]                                \n\t"
291         "pshufh     %[ftmp2],   %[ftmp1],       %[ftmp0]                \n\t"
292         "pmullh     %[ftmp9],   %[ftmp9],       %[ftmp2]                \n\t"
293         "pmullh     %[ftmp10],  %[ftmp10],      %[ftmp2]                \n\t"
294         "paddh      %[ftmp7],   %[ftmp7],       %[ftmp9]                \n\t"
295         "paddh      %[ftmp8],   %[ftmp8],       %[ftmp10]               \n\t"
296         "paddh      %[ftmp7],   %[ftmp7],       %[ftmp11]               \n\t"
297         "paddh      %[ftmp8],   %[ftmp8],       %[ftmp12]               \n\t"
298         "paddh      %[ftmp7],   %[ftmp7],       %[ftmp2]                \n\t"
299         "paddh      %[ftmp8],   %[ftmp8],       %[ftmp2]                \n\t"
300         "psrah      %[ftmp7],   %[ftmp7],       %[ftmp1]                \n\t"
301         "psrah      %[ftmp8],   %[ftmp8],       %[ftmp1]                \n\t"
302         "packushb   %[ftmp5],   %[ftmp7],       %[ftmp8]                \n\t"
303         "biadd      %[ftmp4],   %[ftmp5]                                \n\t"
304         "mfc1       %[dc2],     %[ftmp4]                                \n\t"
305         : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
306           [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
307           [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
308           [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
309           [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
310           [ftmp10]"=&f"(ftmp[10]),          [ftmp11]"=&f"(ftmp[11]),
311           [ftmp12]"=&f"(ftmp[12]),          [ftmp13]"=&f"(ftmp[13]),
312           [tmp0]"=&r"(tmp[0]),              [dc2]"=r"(dc2)
313         : [srcA]"r"((mips_reg)(src-stride-1)),
314           [src0]"r"((mips_reg)(src-stride)),
315           [src1]"r"((mips_reg)(src-stride+1)),
316           [has_topleft]"r"(has_topleft),    [has_topright]"r"(has_topright)
317         : "memory"
318     );
319
320     dc1 = l0+l1+l2+l3+l4+l5+l6+l7;
321     dc = ((dc1+dc2+8)>>4)*0x01010101U;
322
323     __asm__ volatile (
324         "dli        %[tmp0],    0x02                                    \n\t"
325         "punpcklwd  %[ftmp0],   %[dc],          %[dc]                   \n\t"
326         "1:                                                             \n\t"
327         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
328         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
329         "gssdxc1    %[ftmp0],   0x00(%[src],    %[stride])              \n\t"
330         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
331         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
332         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
333         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
334         "gssdxc1    %[ftmp0],   0x00(%[src],    %[stride])              \n\t"
335         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
336         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
337         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
338         "bnez       %[tmp0],    1b                                      \n\t"
339         : [ftmp0]"=&f"(ftmp[0]),            [tmp0]"=&r"(tmp[0]),
340           [src]"+&r"(src)
341         : [dc]"f"(dc),                      [stride]"r"((mips_reg)stride)
342         : "memory"
343     );
344 }
345
346 void ff_pred8x8l_vertical_8_mmi(uint8_t *src, int has_topleft,
347         int has_topright, ptrdiff_t stride)
348 {
349     double ftmp[12];
350     mips_reg tmp[1];
351
352     __asm__ volatile (
353         "xor        %[ftmp0],   %[ftmp0],       %[ftmp0]                \n\t"
354         "gsldlc1    %[ftmp3],   0x07(%[srcA])                           \n\t"
355         "gsldrc1    %[ftmp3],   0x00(%[srcA])                           \n\t"
356         "gsldlc1    %[ftmp4],   0x07(%[src0])                           \n\t"
357         "gsldrc1    %[ftmp4],   0x00(%[src0])                           \n\t"
358         "gsldlc1    %[ftmp5],   0x07(%[src1])                           \n\t"
359         "gsldrc1    %[ftmp5],   0x00(%[src1])                           \n\t"
360         "punpcklbh  %[ftmp6],   %[ftmp3],       %[ftmp0]                \n\t"
361         "punpckhbh  %[ftmp7],   %[ftmp3],       %[ftmp0]                \n\t"
362         "punpcklbh  %[ftmp8],   %[ftmp4],       %[ftmp0]                \n\t"
363         "punpckhbh  %[ftmp9],   %[ftmp4],       %[ftmp0]                \n\t"
364         "punpcklbh  %[ftmp10],  %[ftmp5],       %[ftmp0]                \n\t"
365         "punpckhbh  %[ftmp11],  %[ftmp5],       %[ftmp0]                \n\t"
366         "bnez       %[has_topleft],             1f                      \n\t"
367         "pinsrh_0   %[ftmp6],   %[ftmp6],       %[ftmp8]                \n\t"
368
369         "1:                                                             \n\t"
370         "bnez       %[has_topright],            2f                      \n\t"
371         "pinsrh_3   %[ftmp11],  %[ftmp11],      %[ftmp9]                \n\t"
372
373         "2:                                                             \n\t"
374         "dli        %[tmp0],    0x02                                    \n\t"
375         "mtc1       %[tmp0],    %[ftmp1]                                \n\t"
376         "pshufh     %[ftmp2],   %[ftmp1],       %[ftmp0]                \n\t"
377         "pmullh     %[ftmp8],   %[ftmp8],       %[ftmp2]                \n\t"
378         "pmullh     %[ftmp9],   %[ftmp9],       %[ftmp2]                \n\t"
379         "paddh      %[ftmp6],   %[ftmp6],       %[ftmp8]                \n\t"
380         "paddh      %[ftmp7],   %[ftmp7],       %[ftmp9]                \n\t"
381         "paddh      %[ftmp6],   %[ftmp6],       %[ftmp10]               \n\t"
382         "paddh      %[ftmp7],   %[ftmp7],       %[ftmp11]               \n\t"
383         "paddh      %[ftmp6],   %[ftmp6],       %[ftmp2]                \n\t"
384         "paddh      %[ftmp7],   %[ftmp7],       %[ftmp2]                \n\t"
385         "psrah      %[ftmp6],   %[ftmp6],       %[ftmp1]                \n\t"
386         "psrah      %[ftmp7],   %[ftmp7],       %[ftmp1]                \n\t"
387         "packushb   %[ftmp4],   %[ftmp6],       %[ftmp7]                \n\t"
388         "sdc1       %[ftmp4],   0x00(%[src])                            \n\t"
389         : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
390           [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
391           [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
392           [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
393           [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
394           [ftmp10]"=&f"(ftmp[10]),          [ftmp11]"=&f"(ftmp[11]),
395           [tmp0]"=&r"(tmp[0]),
396           [src]"=r"(src)
397         : [srcA]"r"((mips_reg)(src-stride-1)),
398           [src0]"r"((mips_reg)(src-stride)),
399           [src1]"r"((mips_reg)(src-stride+1)),
400           [has_topleft]"r"(has_topleft),    [has_topright]"r"(has_topright)
401         : "memory"
402     );
403
404     __asm__ volatile (
405         "dli        %[tmp0],    0x02                                    \n\t"
406         "1:                                                             \n\t"
407         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
408         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
409         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
410         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
411         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
412         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
413         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
414         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
415         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
416         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
417         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
418         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
419         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
420         "bnez       %[tmp0],    1b                                      \n\t"
421         : [ftmp0]"=&f"(ftmp[0]),            [tmp0]"=&r"(tmp[0]),
422           [src]"+&r"(src)
423         : [stride]"r"((mips_reg)stride)
424         : "memory"
425     );
426 }
427
428 void ff_pred4x4_dc_8_mmi(uint8_t *src, const uint8_t *topright,
429         ptrdiff_t stride)
430 {
431     const int dc = (src[-stride] + src[1-stride] + src[2-stride]
432                  + src[3-stride] + src[-1+0*stride] + src[-1+1*stride]
433                  + src[-1+2*stride] + src[-1+3*stride] + 4) >>3;
434     uint64_t tmp[2];
435     mips_reg addr[1];
436
437     __asm__ volatile (
438         PTR_ADDU   "%[tmp0],    %[dc],          $0                      \n\t"
439         "dmul       %[tmp1],    %[tmp0],        %[ff_pb_1]              \n\t"
440         "xor        %[addr0],   %[addr0],       %[addr0]                \n\t"
441         "gsswx      %[tmp1],    0x00(%[src],    %[addr0])               \n\t"
442         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
443         "gsswx      %[tmp1],    0x00(%[src],    %[addr0])               \n\t"
444         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
445         "gsswx      %[tmp1],    0x00(%[src],    %[addr0])               \n\t"
446         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
447         "gsswx      %[tmp1],    0x00(%[src],    %[addr0])               \n\t"
448         : [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
449           [addr0]"=&r"(addr[0])
450         : [src]"r"((mips_reg)src),          [stride]"r"((mips_reg)stride),
451           [dc]"r"(dc),                      [ff_pb_1]"r"(ff_pb_1)
452         : "memory"
453     );
454 }
455
456 void ff_pred8x8_vertical_8_mmi(uint8_t *src, ptrdiff_t stride)
457 {
458     uint64_t tmp[2];
459     mips_reg addr[2];
460
461     __asm__ volatile (
462         PTR_SUBU   "%[addr0],   %[src],         %[stride]               \n\t"
463         PTR_ADDU   "%[addr1],   %[src],         $0                      \n\t"
464         "ldl        %[tmp0],    0x07(%[addr0])                          \n\t"
465         "ldr        %[tmp0],    0x00(%[addr0])                          \n\t"
466         "dli        %[tmp1],    0x04                                    \n\t"
467         "1:                                                             \n\t"
468         "sdl        %[tmp0],    0x07(%[addr1])                          \n\t"
469         "sdr        %[tmp0],    0x00(%[addr1])                          \n\t"
470         PTR_ADDU   "%[addr1],   %[stride]                               \n\t"
471         "sdl        %[tmp0],    0x07(%[addr1])                          \n\t"
472         "sdr        %[tmp0],    0x00(%[addr1])                          \n\t"
473         "daddi      %[tmp1],    -0x01                                   \n\t"
474         PTR_ADDU   "%[addr1],   %[stride]                               \n\t"
475         "bnez       %[tmp1],    1b                                      \n\t"
476         : [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
477           [addr0]"=&r"(addr[0]),            [addr1]"=&r"(addr[1])
478         : [src]"r"((mips_reg)src),          [stride]"r"((mips_reg)stride)
479         : "memory"
480     );
481 }
482
483 void ff_pred8x8_horizontal_8_mmi(uint8_t *src, ptrdiff_t stride)
484 {
485     uint64_t tmp[3];
486     mips_reg addr[2];
487
488     __asm__ volatile (
489         PTR_ADDI   "%[addr0],   %[src],         -0x01                   \n\t"
490         PTR_ADDU   "%[addr1],   %[src],         $0                      \n\t"
491         "dli        %[tmp0],    0x04                                    \n\t"
492         "1:                                                             \n\t"
493         "lbu        %[tmp1],    0x00(%[addr0])                          \n\t"
494         "dmul       %[tmp2],    %[tmp1],        %[ff_pb_1]              \n\t"
495         "swl        %[tmp2],    0x07(%[addr1])                          \n\t"
496         "swr        %[tmp2],    0x00(%[addr1])                          \n\t"
497         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
498         PTR_ADDU   "%[addr1],   %[addr1],       %[stride]               \n\t"
499         "lbu        %[tmp1],    0x00(%[addr0])                          \n\t"
500         "dmul       %[tmp2],    %[tmp1],        %[ff_pb_1]              \n\t"
501         "swl        %[tmp2],    0x07(%[addr1])                          \n\t"
502         "swr        %[tmp2],    0x00(%[addr1])                          \n\t"
503         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
504         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
505         PTR_ADDU   "%[addr1],   %[addr1],       %[stride]               \n\t"
506         "bnez       %[tmp0],    1b                                      \n\t"
507         : [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
508           [tmp2]"=&r"(tmp[2]),
509           [addr0]"=&r"(addr[0]),            [addr1]"=&r"(addr[1])
510         : [src]"r"((mips_reg)src),          [stride]"r"((mips_reg)stride),
511           [ff_pb_1]"r"(ff_pb_1)
512         : "memory"
513     );
514 }
515
516 void ff_pred8x8_top_dc_8_mmi(uint8_t *src, ptrdiff_t stride)
517 {
518     double ftmp[4];
519     uint64_t tmp[1];
520     mips_reg addr[1];
521
522     __asm__ volatile (
523         "dli        %[tmp0],    0x02                                    \n\t"
524         "xor        %[ftmp0],   %[ftmp0],       %[ftmp0]                \n\t"
525         PTR_SUBU   "%[addr0],   %[src],         %[stride]               \n\t"
526         "gsldlc1    %[ftmp1],   0x07(%[addr0])                          \n\t"
527         "gsldrc1    %[ftmp1],   0x00(%[addr0])                          \n\t"
528         "punpcklbh  %[ftmp2],   %[ftmp1],       %[ftmp0]                \n\t"
529         "punpckhbh  %[ftmp3],   %[ftmp1],       %[ftmp0]                \n\t"
530         "biadd      %[ftmp2],   %[ftmp2]                                \n\t"
531         "biadd      %[ftmp3],   %[ftmp3]                                \n\t"
532         "mtc1       %[tmp0],    %[ftmp1]                                \n\t"
533         "pshufh     %[ftmp2],   %[ftmp2],       %[ftmp0]                \n\t"
534         "pshufh     %[ftmp3],   %[ftmp3],       %[ftmp0]                \n\t"
535         "pshufh     %[ftmp1],   %[ftmp1],       %[ftmp0]                \n\t"
536         "paddush    %[ftmp2],   %[ftmp2],       %[ftmp1]                \n\t"
537         "paddush    %[ftmp3],   %[ftmp3],       %[ftmp1]                \n\t"
538         "mtc1       %[tmp0],    %[ftmp1]                                \n\t"
539         "psrlh      %[ftmp2],   %[ftmp2],       %[ftmp1]                \n\t"
540         "psrlh      %[ftmp3],   %[ftmp3],       %[ftmp1]                \n\t"
541         "packushb   %[ftmp1],   %[ftmp2],       %[ftmp3]                \n\t"
542         "gssdlc1    %[ftmp1],   0x07(%[src])                            \n\t"
543         "gssdrc1    %[ftmp1],   0x00(%[src])                            \n\t"
544         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
545         "gssdlc1    %[ftmp1],   0x07(%[src])                            \n\t"
546         "gssdrc1    %[ftmp1],   0x00(%[src])                            \n\t"
547         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
548         "gssdlc1    %[ftmp1],   0x07(%[src])                            \n\t"
549         "gssdrc1    %[ftmp1],   0x00(%[src])                            \n\t"
550         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
551         "gssdlc1    %[ftmp1],   0x07(%[src])                            \n\t"
552         "gssdrc1    %[ftmp1],   0x00(%[src])                            \n\t"
553         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
554         "gssdlc1    %[ftmp1],   0x07(%[src])                            \n\t"
555         "gssdrc1    %[ftmp1],   0x00(%[src])                            \n\t"
556         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
557         "gssdlc1    %[ftmp1],   0x07(%[src])                            \n\t"
558         "gssdrc1    %[ftmp1],   0x00(%[src])                            \n\t"
559         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
560         "gssdlc1    %[ftmp1],   0x07(%[src])                            \n\t"
561         "gssdrc1    %[ftmp1],   0x00(%[src])                            \n\t"
562         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
563         "gssdlc1    %[ftmp1],   0x07(%[src])                            \n\t"
564         "gssdrc1    %[ftmp1],   0x00(%[src])                            \n\t"
565         : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
566           [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
567           [tmp0]"=&r"(tmp[0]),
568           [addr0]"=&r"(addr[0]),
569           [src]"+&r"(src)
570         : [stride]"r"((mips_reg)stride)
571         : "memory"
572     );
573 }
574
575 void ff_pred8x8_dc_8_mmi(uint8_t *src, ptrdiff_t stride)
576 {
577     double ftmp[5];
578     mips_reg addr[7];
579
580     __asm__ volatile (
581         "negu       %[addr0],   %[stride]                               \n\t"
582         PTR_ADDU   "%[addr0],   %[addr0],       %[src]                  \n\t"
583         PTR_ADDIU  "%[addr1],   %[addr0],       0x04                    \n\t"
584         "lbu        %[addr2],   0x00(%[addr0])                          \n\t"
585         PTR_ADDU   "%[addr3],   $0,             %[addr2]                \n\t"
586         PTR_ADDIU  "%[addr0],   0x01                                    \n\t"
587         "lbu        %[addr2],   0x00(%[addr1])                          \n\t"
588         PTR_ADDU   "%[addr4],   $0,             %[addr2]                \n\t"
589         PTR_ADDIU  "%[addr1],   0x01                                    \n\t"
590         "lbu        %[addr2],   0x00(%[addr0])                          \n\t"
591         PTR_ADDU   "%[addr3],   %[addr3],       %[addr2]                \n\t"
592         PTR_ADDIU  "%[addr0],   0x01                                    \n\t"
593         "lbu        %[addr2],   0x00(%[addr1])                          \n\t"
594         PTR_ADDU   "%[addr4],   %[addr4],       %[addr2]                \n\t"
595         PTR_ADDIU  "%[addr1],   0x01                                    \n\t"
596         "lbu        %[addr2],   0x00(%[addr0])                          \n\t"
597         PTR_ADDU   "%[addr3],   %[addr3],       %[addr2]                \n\t"
598         PTR_ADDIU  "%[addr0],   0x01                                    \n\t"
599         "lbu        %[addr2],   0x00(%[addr1])                          \n\t"
600         PTR_ADDU   "%[addr4],   %[addr4],       %[addr2]                \n\t"
601         PTR_ADDIU  "%[addr1],   0x01                                    \n\t"
602         "lbu        %[addr2],   0x00(%[addr0])                          \n\t"
603         PTR_ADDU   "%[addr3],   %[addr3],       %[addr2]                \n\t"
604         PTR_ADDIU  "%[addr0],   0x01                                    \n\t"
605         "lbu        %[addr2],   0x00(%[addr1])                          \n\t"
606         PTR_ADDU   "%[addr4],   %[addr4],       %[addr2]                \n\t"
607         PTR_ADDIU  "%[addr1],   0x01                                    \n\t"
608         "dli        %[addr2],  -0x01                                    \n\t"
609         PTR_ADDU   "%[addr2],   %[addr2],       %[src]                  \n\t"
610         "lbu        %[addr1],   0x00(%[addr2])                          \n\t"
611         PTR_ADDU   "%[addr5],   $0,             %[addr1]                \n\t"
612         PTR_ADDU   "%[addr2],   %[addr2],       %[stride]               \n\t"
613         "lbu        %[addr1],   0x00(%[addr2])                          \n\t"
614         PTR_ADDU   "%[addr5],   %[addr5],       %[addr1]                \n\t"
615         PTR_ADDU   "%[addr2],   %[addr2],       %[stride]               \n\t"
616         "lbu        %[addr1],   0x00(%[addr2])                          \n\t"
617         PTR_ADDU   "%[addr5],   %[addr5],       %[addr1]                \n\t"
618         PTR_ADDU   "%[addr2],   %[addr2],       %[stride]               \n\t"
619         "lbu        %[addr1],   0x00(%[addr2])                          \n\t"
620         PTR_ADDU   "%[addr5],   %[addr5],       %[addr1]                \n\t"
621         PTR_ADDU   "%[addr2],   %[addr2],       %[stride]               \n\t"
622         "lbu        %[addr1],   0x00(%[addr2])                          \n\t"
623         PTR_ADDU   "%[addr6],   $0,             %[addr1]                \n\t"
624         PTR_ADDU   "%[addr2],   %[addr2],       %[stride]               \n\t"
625         "lbu        %[addr1],   0x00(%[addr2])                          \n\t"
626         PTR_ADDU   "%[addr6],   %[addr6],       %[addr1]                \n\t"
627         PTR_ADDU   "%[addr2],   %[addr2],       %[stride]               \n\t"
628         "lbu        %[addr1],   0x00(%[addr2])                          \n\t"
629         PTR_ADDU   "%[addr6],   %[addr6],       %[addr1]                \n\t"
630         PTR_ADDU   "%[addr2],   %[addr2],       %[stride]               \n\t"
631         "lbu        %[addr1],   0x00(%[addr2])                          \n\t"
632         PTR_ADDU   "%[addr6],   %[addr6],       %[addr1]                \n\t"
633         PTR_ADDU   "%[addr3],   %[addr3],       %[addr5]                \n\t"
634         PTR_ADDIU  "%[addr3],   %[addr3],       0x04                    \n\t"
635         PTR_ADDIU  "%[addr4],   %[addr4],       0x02                    \n\t"
636         PTR_ADDIU  "%[addr1],   %[addr6],       0x02                    \n\t"
637         PTR_ADDU   "%[addr2],   %[addr4],       %[addr1]                \n\t"
638         PTR_SRL    "%[addr3],   0x03                                    \n\t"
639         PTR_SRL    "%[addr4],   0x02                                    \n\t"
640         PTR_SRL    "%[addr1],   0x02                                    \n\t"
641         PTR_SRL    "%[addr2],   0x03                                    \n\t"
642         "xor        %[ftmp0],   %[ftmp0],       %[ftmp0]                \n\t"
643         "dmtc1      %[addr3],   %[ftmp1]                                \n\t"
644         "pshufh     %[ftmp1],   %[ftmp1],       %[ftmp0]                \n\t"
645         "dmtc1      %[addr4],   %[ftmp2]                                \n\t"
646         "pshufh     %[ftmp2],   %[ftmp2],       %[ftmp0]                \n\t"
647         "dmtc1      %[addr1],   %[ftmp3]                                \n\t"
648         "pshufh     %[ftmp3],   %[ftmp3],       %[ftmp0]                \n\t"
649         "dmtc1      %[addr2],   %[ftmp4]                                \n\t"
650         "pshufh     %[ftmp4],   %[ftmp4],       %[ftmp0]                \n\t"
651         "packushb   %[ftmp1],   %[ftmp1],       %[ftmp2]                \n\t"
652         "packushb   %[ftmp2],   %[ftmp3],       %[ftmp4]                \n\t"
653         PTR_ADDU   "%[addr0],   $0,             %[src]                  \n\t"
654         "sdc1       %[ftmp1],   0x00(%[addr0])                          \n\t"
655         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
656         "sdc1       %[ftmp1],   0x00(%[addr0])                          \n\t"
657         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
658         "sdc1       %[ftmp1],   0x00(%[addr0])                          \n\t"
659         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
660         "sdc1       %[ftmp1],   0x00(%[addr0])                          \n\t"
661         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
662         "sdc1       %[ftmp2],   0x00(%[addr0])                          \n\t"
663         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
664         "sdc1       %[ftmp2],   0x00(%[addr0])                          \n\t"
665         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
666         "sdc1       %[ftmp2],   0x00(%[addr0])                          \n\t"
667         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
668         "sdc1       %[ftmp2],   0x00(%[addr0])                          \n\t"
669         : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
670           [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
671           [ftmp4]"=&f"(ftmp[4]),
672           [addr0]"=&r"(addr[0]),            [addr1]"=&r"(addr[1]),
673           [addr2]"=&r"(addr[2]),            [addr3]"=&r"(addr[3]),
674           [addr4]"=&r"(addr[4]),            [addr5]"=&r"(addr[5]),
675           [addr6]"=&r"(addr[6])
676         : [src]"r"((mips_reg)src),          [stride]"r"((mips_reg)stride)
677         : "memory"
678     );
679 }
680
681 void ff_pred8x16_vertical_8_mmi(uint8_t *src, ptrdiff_t stride)
682 {
683     double ftmp[1];
684     uint64_t tmp[1];
685
686     __asm__ volatile (
687         "gsldlc1    %[ftmp0],   0x07(%[srcA])                           \n\t"
688         "gsldrc1    %[ftmp0],   0x00(%[srcA])                           \n\t"
689         "dli        %[tmp0],    0x04                                    \n\t"
690         "1:                                                             \n\t"
691         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
692         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
693         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
694         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
695         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
696         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
697         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
698         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
699         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
700         "gssdlc1    %[ftmp0],   0x07(%[src])                            \n\t"
701         "gssdrc1    %[ftmp0],   0x00(%[src])                            \n\t"
702         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
703         PTR_ADDU   "%[src],     %[src],         %[stride]               \n\t"
704         "bnez       %[tmp0],    1b                                      \n\t"
705         : [ftmp0]"=&f"(ftmp[0]),
706           [tmp0]"=&r"(tmp[0]),
707           [src]"+&r"(src)
708         : [stride]"r"((mips_reg)stride),    [srcA]"r"((mips_reg)(src-stride))
709         : "memory"
710     );
711 }
712
713 void ff_pred8x16_horizontal_8_mmi(uint8_t *src, ptrdiff_t stride)
714 {
715     uint64_t tmp[3];
716     mips_reg addr[2];
717
718     __asm__ volatile (
719         PTR_ADDI   "%[addr0],   %[src],         -0x01                   \n\t"
720         PTR_ADDU   "%[addr1],   %[src],         $0                      \n\t"
721         "dli        %[tmp0],    0x08                                    \n\t"
722         "1:                                                             \n\t"
723         "lbu        %[tmp1],    0x00(%[addr0])                          \n\t"
724         "dmul       %[tmp2],    %[tmp1],        %[ff_pb_1]              \n\t"
725         "swl        %[tmp2],    0x07(%[addr1])                          \n\t"
726         "swr        %[tmp2],    0x00(%[addr1])                          \n\t"
727         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
728         PTR_ADDU   "%[addr1],   %[addr1],       %[stride]               \n\t"
729         "lbu        %[tmp1],    0x00(%[addr0])                          \n\t"
730         "dmul       %[tmp2],    %[tmp1],        %[ff_pb_1]              \n\t"
731         "swl        %[tmp2],    0x07(%[addr1])                          \n\t"
732         "swr        %[tmp2],    0x00(%[addr1])                          \n\t"
733         "daddi      %[tmp0],    %[tmp0],        -0x01                   \n\t"
734         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
735         PTR_ADDU   "%[addr1],   %[addr1],       %[stride]               \n\t"
736         "bnez       %[tmp0],    1b                                      \n\t"
737         : [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
738           [tmp2]"=&r"(tmp[2]),
739           [addr0]"=&r"(addr[0]),            [addr1]"=&r"(addr[1])
740         : [src]"r"((mips_reg)src),          [stride]"r"((mips_reg)stride),
741           [ff_pb_1]"r"(ff_pb_1)
742         : "memory"
743     );
744 }
745
746 static inline void pred16x16_plane_compat_mmi(uint8_t *src, int stride,
747         const int svq3, const int rv40)
748 {
749     double ftmp[11];
750     uint64_t tmp[7];
751     mips_reg addr[1];
752
753     __asm__ volatile(
754         PTR_SUBU   "%[addr0],   %[src],         %[stride]               \n\t"
755         "dli        %[tmp2],    0x20                                    \n\t"
756         "dmtc1      %[tmp2],    %[ftmp4]                                \n\t"
757         "gsldlc1    %[ftmp0],   0x06(%[addr0])                          \n\t"
758         "gsldlc1    %[ftmp2],   0x0f(%[addr0])                          \n\t"
759         "gsldrc1    %[ftmp0],   -0x01(%[addr0])                         \n\t"
760         "gsldrc1    %[ftmp2],   0x08(%[addr0])                          \n\t"
761         "dsrl       %[ftmp1],   %[ftmp0],       %[ftmp4]                \n\t"
762         "dsrl       %[ftmp3],   %[ftmp2],       %[ftmp4]                \n\t"
763         "xor        %[ftmp4],   %[ftmp4],       %[ftmp4]                \n\t"
764         "punpcklbh  %[ftmp0],   %[ftmp0],       %[ftmp4]                \n\t"
765         "punpcklbh  %[ftmp1],   %[ftmp1],       %[ftmp4]                \n\t"
766         "punpcklbh  %[ftmp2],   %[ftmp2],       %[ftmp4]                \n\t"
767         "punpcklbh  %[ftmp3],   %[ftmp3],       %[ftmp4]                \n\t"
768         "pmullh     %[ftmp0],   %[ftmp0],       %[ff_pw_m8tom5]         \n\t"
769         "pmullh     %[ftmp1],   %[ftmp1],       %[ff_pw_m4tom1]         \n\t"
770         "pmullh     %[ftmp2],   %[ftmp2],       %[ff_pw_1to4]           \n\t"
771         "pmullh     %[ftmp3],   %[ftmp3],       %[ff_pw_5to8]           \n\t"
772         "paddsh     %[ftmp0],   %[ftmp0],       %[ftmp2]                \n\t"
773         "paddsh     %[ftmp1],   %[ftmp1],       %[ftmp3]                \n\t"
774         "paddsh     %[ftmp0],   %[ftmp0],       %[ftmp1]                \n\t"
775         "dli        %[tmp2],    0x0e                                    \n\t"
776         "dmtc1      %[tmp2],    %[ftmp4]                                \n\t"
777         "pshufh     %[ftmp1],   %[ftmp0],       %[ftmp4]                \n\t"
778         "paddsh     %[ftmp0],   %[ftmp0],       %[ftmp1]                \n\t"
779         "dli        %[tmp2],    0x01                                    \n\t"
780         "dmtc1      %[tmp2],    %[ftmp4]                                \n\t"
781         "pshufh     %[ftmp1],   %[ftmp0],       %[ftmp4]                \n\t"
782         "paddsh     %[ftmp5],   %[ftmp0],       %[ftmp1]                \n\t"
783
784         PTR_ADDIU  "%[addr0],   %[src],         -0x01                   \n\t"
785         PTR_SUBU   "%[addr0],   %[addr0],       %[stride]               \n\t"
786         "lbu        %[tmp2],    0x00(%[addr0])                          \n\t"
787         "lbu        %[tmp6],    0x10(%[addr0])                          \n\t"
788         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
789         "lbu        %[tmp3],    0x00(%[addr0])                          \n\t"
790         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
791         "lbu        %[tmp4],    0x00(%[addr0])                          \n\t"
792         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
793         "lbu        %[tmp5],    0x00(%[addr0])                          \n\t"
794         "dsll       %[tmp3],    %[tmp3],        0x10                    \n\t"
795         "dsll       %[tmp4],    %[tmp4],        0x20                    \n\t"
796         "dsll       %[tmp5],    %[tmp5],        0x30                    \n\t"
797         "or         %[tmp4],    %[tmp4],        %[tmp5]                 \n\t"
798         "or         %[tmp2],    %[tmp2],        %[tmp3]                 \n\t"
799         "or         %[tmp2],    %[tmp2],        %[tmp4]                 \n\t"
800         "dmtc1      %[tmp2],    %[ftmp0]                                \n\t"
801
802         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
803         "lbu        %[tmp2],    0x00(%[addr0])                          \n\t"
804         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
805         "lbu        %[tmp3],    0x00(%[addr0])                          \n\t"
806         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
807         "lbu        %[tmp4],    0x00(%[addr0])                          \n\t"
808         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
809         "lbu        %[tmp5],    0x00(%[addr0])                          \n\t"
810         "dsll       %[tmp3],    %[tmp3],        0x10                    \n\t"
811         "dsll       %[tmp4],    %[tmp4],        0x20                    \n\t"
812         "dsll       %[tmp5],    %[tmp5],        0x30                    \n\t"
813         "or         %[tmp4],    %[tmp4],        %[tmp5]                 \n\t"
814         "or         %[tmp2],    %[tmp2],        %[tmp3]                 \n\t"
815         "or         %[tmp2],    %[tmp2],        %[tmp4]                 \n\t"
816         "dmtc1      %[tmp2],    %[ftmp1]                                \n\t"
817
818         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
819         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
820         "lbu        %[tmp2],    0x00(%[addr0])                          \n\t"
821         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
822         "lbu        %[tmp3],    0x00(%[addr0])                          \n\t"
823         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
824         "lbu        %[tmp4],    0x00(%[addr0])                          \n\t"
825         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
826         "lbu        %[tmp5],    0x00(%[addr0])                          \n\t"
827         "dsll       %[tmp3],    %[tmp3],        0x10                    \n\t"
828         "dsll       %[tmp4],    %[tmp4],        0x20                    \n\t"
829         "dsll       %[tmp5],    %[tmp5],        0x30                    \n\t"
830         "or         %[tmp4],    %[tmp4],        %[tmp5]                 \n\t"
831         "or         %[tmp2],    %[tmp2],        %[tmp3]                 \n\t"
832         "or         %[tmp2],    %[tmp2],        %[tmp4]                 \n\t"
833         "dmtc1      %[tmp2],    %[ftmp2]                                \n\t"
834
835         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
836         "lbu        %[tmp2],    0x00(%[addr0])                          \n\t"
837         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
838         "lbu        %[tmp3],    0x00(%[addr0])                          \n\t"
839         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
840         "lbu        %[tmp4],    0x00(%[addr0])                          \n\t"
841         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
842         "lbu        %[tmp5],    0x00(%[addr0])                          \n\t"
843         "daddu      %[tmp6],    %[tmp6],        %[tmp5]                 \n\t"
844         "daddiu     %[tmp6],    %[tmp6],        0x01                    \n\t"
845         "dsll       %[tmp6],    %[tmp6],        0x04                    \n\t"
846
847         "dsll       %[tmp3],    %[tmp3],        0x10                    \n\t"
848         "dsll       %[tmp4],    %[tmp4],        0x20                    \n\t"
849         "dsll       %[tmp5],    %[tmp5],        0x30                    \n\t"
850         "or         %[tmp4],    %[tmp4],        %[tmp5]                 \n\t"
851         "or         %[tmp2],    %[tmp2],        %[tmp3]                 \n\t"
852         "or         %[tmp2],    %[tmp2],        %[tmp4]                 \n\t"
853         "dmtc1      %[tmp2],    %[ftmp3]                                \n\t"
854
855         "pmullh     %[ftmp0],   %[ftmp0],       %[ff_pw_m8tom5]         \n\t"
856         "pmullh     %[ftmp1],   %[ftmp1],       %[ff_pw_m4tom1]         \n\t"
857         "pmullh     %[ftmp2],   %[ftmp2],       %[ff_pw_1to4]           \n\t"
858         "pmullh     %[ftmp3],   %[ftmp3],       %[ff_pw_5to8]           \n\t"
859         "paddsh     %[ftmp0],   %[ftmp0],       %[ftmp2]                \n\t"
860         "paddsh     %[ftmp1],   %[ftmp1],       %[ftmp3]                \n\t"
861         "paddsh     %[ftmp0],   %[ftmp0],       %[ftmp1]                \n\t"
862         "dli        %[tmp2],    0x0e                                    \n\t"
863         "dmtc1      %[tmp2],    %[ftmp4]                                \n\t"
864         "pshufh     %[ftmp1],   %[ftmp0],       %[ftmp4]                \n\t"
865         "paddsh     %[ftmp0],   %[ftmp0],       %[ftmp1]                \n\t"
866
867         "dli        %[tmp2],    0x01                                    \n\t"
868         "dmtc1      %[tmp2],    %[ftmp4]                                \n\t"
869         "pshufh     %[ftmp1],   %[ftmp0],       %[ftmp4]                \n\t"
870         "paddsh     %[ftmp6],   %[ftmp0],       %[ftmp1]                \n\t"
871
872         "dmfc1      %[tmp0],    %[ftmp5]                                \n\t"
873         "dsll       %[tmp0],    %[tmp0],        0x30                    \n\t"
874         "dsra       %[tmp0],    %[tmp0],        0x30                    \n\t"
875         "dmfc1      %[tmp1],    %[ftmp6]                                \n\t"
876         "dsll       %[tmp1],    %[tmp1],        0x30                    \n\t"
877         "dsra       %[tmp1],    %[tmp1],        0x30                    \n\t"
878
879         "beqz       %[svq3],    1f                                      \n\t"
880         "dli        %[tmp2],    0x04                                    \n\t"
881         "ddiv       %[tmp0],    %[tmp0],        %[tmp2]                 \n\t"
882         "ddiv       %[tmp1],    %[tmp1],        %[tmp2]                 \n\t"
883         "dli        %[tmp2],    0x05                                    \n\t"
884         "dmul       %[tmp0],    %[tmp0],        %[tmp2]                 \n\t"
885         "dmul       %[tmp1],    %[tmp1],        %[tmp2]                 \n\t"
886         "dli        %[tmp2],    0x10                                    \n\t"
887         "ddiv       %[tmp0],    %[tmp0],        %[tmp2]                 \n\t"
888         "ddiv       %[tmp1],    %[tmp1],        %[tmp2]                 \n\t"
889         "daddu      %[tmp2],    %[tmp0],        $0                      \n\t"
890         "daddu      %[tmp0],    %[tmp1],        $0                      \n\t"
891         "daddu      %[tmp1],    %[tmp2],        $0                      \n\t"
892         "b          2f                                                  \n\t"
893
894         "1:                                                             \n\t"
895         "beqz       %[rv40],    1f                                      \n\t"
896         "dsra       %[tmp2],    %[tmp0],        0x02                    \n\t"
897         "daddu      %[tmp0],    %[tmp0],        %[tmp2]                 \n\t"
898         "dsra       %[tmp2],    %[tmp1],        0x02                    \n\t"
899         "daddu      %[tmp1],    %[tmp1],        %[tmp2]                 \n\t"
900         "dsra       %[tmp0],    %[tmp0],        0x04                    \n\t"
901         "dsra       %[tmp1],    %[tmp1],        0x04                    \n\t"
902         "b          2f                                                  \n\t"
903
904         "1:                                                             \n\t"
905         "dli        %[tmp2],    0x05                                    \n\t"
906         "dmul       %[tmp0],    %[tmp0],        %[tmp2]                 \n\t"
907         "dmul       %[tmp1],    %[tmp1],        %[tmp2]                 \n\t"
908         "daddiu     %[tmp0],    %[tmp0],        0x20                    \n\t"
909         "daddiu     %[tmp1],    %[tmp1],        0x20                    \n\t"
910         "dsra       %[tmp0],    %[tmp0],        0x06                    \n\t"
911         "dsra       %[tmp1],    %[tmp1],        0x06                    \n\t"
912
913         "2:                                                             \n\t"
914         "daddu      %[tmp3],    %[tmp0],        %[tmp1]                 \n\t"
915         "dli        %[tmp2],    0x07                                    \n\t"
916         "dmul       %[tmp3],    %[tmp3],        %[tmp2]                 \n\t"
917         "dsubu      %[tmp6],    %[tmp6],        %[tmp3]                 \n\t"
918
919         "xor        %[ftmp4],   %[ftmp4],       %[ftmp4]                \n\t"
920         "dmtc1      %[tmp0],    %[ftmp0]                                \n\t"
921         "pshufh     %[ftmp0],   %[ftmp0],       %[ftmp4]                \n\t"
922         "dmtc1      %[tmp1],    %[ftmp5]                                \n\t"
923         "pshufh     %[ftmp5],   %[ftmp5],       %[ftmp4]                \n\t"
924         "dmtc1      %[tmp6],    %[ftmp6]                                \n\t"
925         "pshufh     %[ftmp6],   %[ftmp6],       %[ftmp4]                \n\t"
926         "dli        %[tmp2],    0x05                                    \n\t"
927         "dmtc1      %[tmp2],    %[ftmp7]                                \n\t"
928         "pmullh     %[ftmp1],   %[ff_pw_0to3],  %[ftmp0]                \n\t"
929         "dmtc1      %[ff_pw_4to7],              %[ftmp2]                \n\t"
930         "pmullh     %[ftmp2],   %[ftmp2],       %[ftmp0]                \n\t"
931         "dmtc1      %[ff_pw_8tob],              %[ftmp3]                \n\t"
932         "pmullh     %[ftmp3],   %[ftmp3],       %[ftmp0]                \n\t"
933         "dmtc1      %[ff_pw_ctof],              %[ftmp4]                \n\t"
934         "pmullh     %[ftmp4],   %[ftmp4],       %[ftmp0]                \n\t"
935
936         "dli        %[tmp0],    0x10                                    \n\t"
937         PTR_ADDU   "%[addr0],   %[src],         $0                      \n\t"
938         "1:                                                             \n\t"
939         "paddsh     %[ftmp8],   %[ftmp1],       %[ftmp6]                \n\t"
940         "psrah      %[ftmp8],   %[ftmp8],       %[ftmp7]                \n\t"
941         "paddsh     %[ftmp9],   %[ftmp2],       %[ftmp6]                \n\t"
942         "psrah      %[ftmp9],   %[ftmp9],       %[ftmp7]                \n\t"
943         "packushb   %[ftmp0],   %[ftmp8],       %[ftmp9]                \n\t"
944         "gssdlc1    %[ftmp0],   0x07(%[addr0])                          \n\t"
945         "gssdrc1    %[ftmp0],   0x00(%[addr0])                          \n\t"
946
947         "paddsh     %[ftmp8],   %[ftmp3],       %[ftmp6]                \n\t"
948         "psrah      %[ftmp8],   %[ftmp8],       %[ftmp7]                \n\t"
949         "paddsh     %[ftmp9],   %[ftmp4],       %[ftmp6]                \n\t"
950         "psrah      %[ftmp9],   %[ftmp9],       %[ftmp7]                \n\t"
951         "packushb   %[ftmp0],   %[ftmp8],       %[ftmp9]                \n\t"
952         "gssdlc1    %[ftmp0],   0x0f(%[addr0])                          \n\t"
953         "gssdrc1    %[ftmp0],   0x08(%[addr0])                          \n\t"
954
955         "paddsh     %[ftmp6],   %[ftmp6],       %[ftmp5]                \n\t"
956         PTR_ADDU   "%[addr0],   %[addr0],       %[stride]               \n\t"
957         "daddiu     %[tmp0],    %[tmp0],        -0x01                   \n\t"
958         "bnez       %[tmp0],    1b                                      \n\t"
959         : [ftmp0]"=&f"(ftmp[0]),            [ftmp1]"=&f"(ftmp[1]),
960           [ftmp2]"=&f"(ftmp[2]),            [ftmp3]"=&f"(ftmp[3]),
961           [ftmp4]"=&f"(ftmp[4]),            [ftmp5]"=&f"(ftmp[5]),
962           [ftmp6]"=&f"(ftmp[6]),            [ftmp7]"=&f"(ftmp[7]),
963           [ftmp8]"=&f"(ftmp[8]),            [ftmp9]"=&f"(ftmp[9]),
964           [tmp0]"=&r"(tmp[0]),              [tmp1]"=&r"(tmp[1]),
965           [tmp2]"=&r"(tmp[2]),              [tmp3]"=&r"(tmp[3]),
966           [tmp4]"=&r"(tmp[4]),              [tmp5]"=&r"(tmp[5]),
967           [tmp6]"=&r"(tmp[6]),
968           [addr0]"=&r"(addr[0])
969         : [src]"r"(src),                    [stride]"r"((mips_reg)stride),
970           [svq3]"r"(svq3),                  [rv40]"r"(rv40),
971           [ff_pw_m8tom5]"f"(ff_pw_m8tom5),  [ff_pw_m4tom1]"f"(ff_pw_m4tom1),
972           [ff_pw_1to4]"f"(ff_pw_1to4),      [ff_pw_5to8]"f"(ff_pw_5to8),
973           [ff_pw_0to3]"f"(ff_pw_0to3),      [ff_pw_4to7]"r"(ff_pw_4to7),
974           [ff_pw_8tob]"r"(ff_pw_8tob),      [ff_pw_ctof]"r"(ff_pw_ctof)
975         : "memory"
976     );
977 }
978
979 void ff_pred16x16_plane_h264_8_mmi(uint8_t *src, ptrdiff_t stride)
980 {
981     pred16x16_plane_compat_mmi(src, stride, 0, 0);
982 }
983
984 void ff_pred16x16_plane_svq3_8_mmi(uint8_t *src, ptrdiff_t stride)
985 {
986     pred16x16_plane_compat_mmi(src, stride, 1, 0);
987 }
988
989 void ff_pred16x16_plane_rv40_8_mmi(uint8_t *src, ptrdiff_t stride)
990 {
991     pred16x16_plane_compat_mmi(src, stride, 0, 1);
992 }