avcodec/mips: MSA (MIPS-SIMD-Arch) optimizations for qpel functions
[ffmpeg.git] / libavcodec / mips / qpeldsp_msa.c
1 /*
2  * Copyright (c) 2015 Parag Salasakar (Parag.Salasakar@imgtec.com)
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 #include "libavutil/mips/generic_macros_msa.h"
22 #include "qpeldsp_mips.h"
23
24 #define APPLY_HORIZ_QPEL_FILTER(inp0, inp1, mask, coef0, coef1, coef2)  \
25 ( {                                                                     \
26     v16u8 out, tmp0, tmp1;                                              \
27     v16u8 data0, data1, data2, data3, data4, data5;                     \
28     v8i16 res_r, res_l;                                                 \
29     v8u16 sum0_r, sum1_r, sum2_r, sum3_r;                               \
30     v8u16 sum0_l, sum1_l, sum2_l, sum3_l;                               \
31                                                                         \
32     VSHF_B2_UB(inp0, inp0, inp1, inp1, mask, mask, tmp0, tmp1);         \
33     ILVRL_B2_UH(inp1, inp0, sum0_r, sum0_l);                            \
34     data0 = (v16u8) __msa_sldi_b((v16i8) inp0, (v16i8) tmp0, 15);       \
35     data3 = (v16u8) __msa_sldi_b((v16i8) tmp1, (v16i8) inp1, 1);        \
36     HADD_UB2_UH(sum0_r, sum0_l, sum0_r, sum0_l);                        \
37     ILVRL_B2_UH(data3, data0, sum1_r, sum1_l);                          \
38     data1 = (v16u8) __msa_sldi_b((v16i8) inp0, (v16i8) tmp0, 14);       \
39     data4 = (v16u8) __msa_sldi_b((v16i8) tmp1, (v16i8) inp1, 2);        \
40     sum0_r *= (v8u16) (coef0);                                          \
41     sum0_l *= (v8u16) (coef0);                                          \
42     ILVRL_B2_UH(data4, data1, sum2_r, sum2_l);                          \
43     data2 = (v16u8) __msa_sldi_b((v16i8) inp0, (v16i8) tmp0, 13);       \
44     data5 = (v16u8) __msa_sldi_b((v16i8) tmp1, (v16i8) inp1, 3);        \
45     DPADD_UB2_UH(sum2_r, sum2_l, coef2, coef2, sum0_r, sum0_l);         \
46     ILVRL_B2_UH(data5, data2, sum3_r, sum3_l);                          \
47     HADD_UB2_UH(sum3_r, sum3_l, sum3_r, sum3_l);                        \
48     DPADD_UB2_UH(sum1_r, sum1_l, coef1, coef1, sum3_r, sum3_l);         \
49     res_r = (v8i16) (sum0_r - sum3_r);                                  \
50     res_l = (v8i16) (sum0_l - sum3_l);                                  \
51     SRARI_H2_SH(res_r, res_l, 5);                                       \
52     CLIP_SH2_0_255(res_r, res_l);                                       \
53     out = (v16u8) __msa_pckev_b((v16i8) res_l, (v16i8) res_r);          \
54                                                                         \
55     out;                                                                \
56 } )
57
58 #define APPLY_HORIZ_QPEL_FILTER_8BYTE(inp0, inp1,                       \
59                                       mask0, mask1, mask2, mask3,       \
60                                       coef0, coef1, coef2)              \
61 ( {                                                                     \
62     v16u8 out;                                                          \
63     v8u16 sum0_r, sum1_r, sum2_r, sum3_r;                               \
64     v8u16 sum4_r, sum5_r, sum6_r, sum7_r;                               \
65     v8i16 res0_r, res1_r;                                               \
66                                                                         \
67     VSHF_B2_UH(inp0, inp0, inp1, inp1, mask0, mask0, sum0_r, sum4_r);   \
68     VSHF_B2_UH(inp0, inp0, inp1, inp1, mask3, mask3, sum3_r, sum7_r);   \
69     HADD_UB2_UH(sum3_r, sum7_r, sum3_r, sum7_r);                        \
70     DOTP_UB2_UH(sum0_r, sum4_r, coef0, coef0, sum0_r, sum4_r);          \
71     VSHF_B2_UH(inp0, inp0, inp1, inp1, mask2, mask2, sum2_r, sum6_r);   \
72     VSHF_B2_UH(inp0, inp0, inp1, inp1, mask1, mask1, sum1_r, sum5_r);   \
73     DPADD_UB2_UH(sum2_r, sum6_r, coef2, coef2, sum0_r, sum4_r);         \
74     DPADD_UB2_UH(sum1_r, sum5_r, coef1, coef1, sum3_r, sum7_r);         \
75     res0_r = (v8i16) (sum0_r - sum3_r);                                 \
76     res1_r = (v8i16) (sum4_r - sum7_r);                                 \
77     SRARI_H2_SH(res0_r, res1_r, 5);                                     \
78     CLIP_SH2_0_255(res0_r, res1_r);                                     \
79     out = (v16u8) __msa_pckev_b((v16i8) res1_r, (v16i8) res0_r);        \
80                                                                         \
81     out;                                                                \
82 } )
83
84 #define APPLY_HORIZ_QPEL_FILTER_8BYTE_1ROW(inp0,                        \
85                                            mask0, mask1, mask2, mask3,  \
86                                            coef0, coef1, coef2)         \
87 ( {                                                                     \
88     v16u8 out;                                                          \
89     v8i16 res0_r;                                                       \
90     v8u16 sum0_r, sum1_r, sum2_r, sum3_r;                               \
91                                                                         \
92     VSHF_B2_UH(inp0, inp0, inp0, inp0, mask0, mask3, sum0_r, sum3_r);   \
93     sum3_r = __msa_hadd_u_h((v16u8) sum3_r, (v16u8) sum3_r);            \
94     sum0_r = __msa_dotp_u_h((v16u8) sum0_r, (v16u8) coef0);             \
95     VSHF_B2_UH(inp0, inp0, inp0, inp0, mask2, mask1, sum2_r, sum1_r);   \
96     DPADD_UB2_UH(sum2_r, sum1_r, coef2, coef1, sum0_r, sum3_r);         \
97     res0_r = (v8i16) (sum0_r - sum3_r);                                 \
98     res0_r = __msa_srari_h(res0_r, 5);                                  \
99     res0_r = CLIP_SH_0_255(res0_r);                                     \
100     out = (v16u8) __msa_pckev_b((v16i8) res0_r, (v16i8) res0_r);        \
101                                                                         \
102     out;                                                                \
103 } )
104
105 #define APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE_1ROW(inp0, mask0, mask1,   \
106                                                     mask2, mask3, coef0,  \
107                                                     coef1, coef2)         \
108 ( {                                                                       \
109     v16u8 out;                                                            \
110     v8i16 res0_r;                                                         \
111     v8u16 sum0_r, sum1_r, sum2_r, sum3_r;                                 \
112                                                                           \
113     VSHF_B2_UH(inp0, inp0, inp0, inp0, mask0, mask3, sum0_r, sum3_r);     \
114     sum3_r = __msa_hadd_u_h((v16u8) sum3_r, (v16u8) sum3_r);              \
115     sum0_r = __msa_dotp_u_h((v16u8) sum0_r, (v16u8) coef0);               \
116     VSHF_B2_UH(inp0, inp0, inp0, inp0, mask2, mask1, sum2_r, sum1_r);     \
117     DPADD_UB2_UH(sum2_r, sum1_r, coef2, coef1, sum0_r, sum3_r);           \
118     res0_r = (v8i16) (sum0_r - sum3_r);                                   \
119     res0_r += 15;                                                         \
120     res0_r >>= 5;                                                         \
121     res0_r = CLIP_SH_0_255(res0_r);                                       \
122     out = (v16u8) __msa_pckev_b((v16i8) res0_r, (v16i8) res0_r);          \
123                                                                           \
124     out;                                                                  \
125 } )
126
127 #define APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp0, inp1, mask,              \
128                                          coef0, coef1, coef2)           \
129 ( {                                                                     \
130     v16u8 out, tmp0, tmp1;                                              \
131     v16u8 data0, data1, data2, data3, data4, data5;                     \
132     v8i16 res_r, res_l;                                                 \
133     v8u16 sum0_r, sum1_r, sum2_r, sum3_r;                               \
134     v8u16 sum0_l, sum1_l, sum2_l, sum3_l;                               \
135                                                                         \
136     VSHF_B2_UB(inp0, inp0, inp1, inp1, mask, mask, tmp0, tmp1);         \
137     ILVRL_B2_UH(inp1, inp0, sum0_r, sum0_l);                            \
138     data0 = (v16u8) __msa_sldi_b((v16i8) inp0, (v16i8) tmp0, 15);       \
139     data3 = (v16u8) __msa_sldi_b((v16i8) tmp1, (v16i8) inp1, 1);        \
140     HADD_UB2_UH(sum0_r, sum0_l, sum0_r, sum0_l);                        \
141     ILVRL_B2_UH(data3, data0, sum1_r, sum1_l);                          \
142     data1 = (v16u8) __msa_sldi_b((v16i8) inp0, (v16i8) tmp0, 14);       \
143     data4 = (v16u8) __msa_sldi_b((v16i8) tmp1, (v16i8) inp1, 2);        \
144     sum0_r *= (v8u16) (coef0);                                          \
145     sum0_l *= (v8u16) (coef0);                                          \
146     ILVRL_B2_UH(data4, data1, sum2_r, sum2_l);                          \
147     data2 = (v16u8) __msa_sldi_b((v16i8) inp0, (v16i8) tmp0, 13);       \
148     data5 = (v16u8) __msa_sldi_b((v16i8) tmp1, (v16i8) inp1, 3);        \
149     DPADD_UB2_UH(sum2_r, sum2_l, coef2, coef2, sum0_r, sum0_l);         \
150     ILVRL_B2_UH(data5, data2, sum3_r, sum3_l);                          \
151     HADD_UB2_UH(sum3_r, sum3_l, sum3_r, sum3_l);                        \
152     DPADD_UB2_UH(sum1_r, sum1_l, coef1, coef1, sum3_r, sum3_l);         \
153     res_r = (v8i16) (sum0_r - sum3_r);                                  \
154     res_l = (v8i16) (sum0_l - sum3_l);                                  \
155     res_r += 15;                                                        \
156     res_l += 15;                                                        \
157     res_r >>= 5;                                                        \
158     res_l >>= 5;                                                        \
159     CLIP_SH2_0_255(res_r, res_l);                                       \
160     out = (v16u8) __msa_pckev_b((v16i8) res_l, (v16i8) res_r);          \
161                                                                         \
162     out;                                                                \
163 } )
164
165 #define APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp0, inp1,                  \
166                                                mask0, mask1, mask2, mask3,  \
167                                                coef0, coef1, coef2)         \
168 ( {                                                                         \
169     v16u8 out;                                                              \
170     v8i16 res0_r, res1_r;                                                   \
171     v8u16 sum0_r, sum1_r, sum2_r, sum3_r;                                   \
172     v8u16 sum4_r, sum5_r, sum6_r, sum7_r;                                   \
173                                                                             \
174     VSHF_B2_UH(inp0, inp0, inp1, inp1, mask0, mask0, sum0_r, sum4_r);       \
175     VSHF_B2_UH(inp0, inp0, inp1, inp1, mask3, mask3, sum3_r, sum7_r);       \
176     HADD_UB2_UH(sum3_r, sum7_r, sum3_r, sum7_r);                            \
177     DOTP_UB2_UH(sum0_r, sum4_r, coef0, coef0, sum0_r, sum4_r);              \
178     VSHF_B2_UH(inp0, inp0, inp1, inp1, mask2, mask2, sum2_r, sum6_r);       \
179     VSHF_B2_UH(inp0, inp0, inp1, inp1, mask1, mask1, sum1_r, sum5_r);       \
180     DPADD_UB2_UH(sum2_r, sum6_r, coef2, coef2, sum0_r, sum4_r);             \
181     DPADD_UB2_UH(sum1_r, sum5_r, coef1, coef1, sum3_r, sum7_r);             \
182     res0_r = (v8i16) (sum0_r - sum3_r);                                     \
183     res1_r = (v8i16) (sum4_r - sum7_r);                                     \
184     res0_r += 15;                                                           \
185     res1_r += 15;                                                           \
186     res0_r >>= 5;                                                           \
187     res1_r >>= 5;                                                           \
188     CLIP_SH2_0_255(res0_r, res1_r);                                         \
189     out = (v16u8) __msa_pckev_b((v16i8) res1_r, (v16i8) res0_r);            \
190                                                                             \
191     out;                                                                    \
192 } )
193
194 #define APPLY_VERT_QPEL_FILTER(inp0, inp1, inp2, inp3,                  \
195                                inp4, inp5, inp6, inp7,                  \
196                                coef0, coef1, coef2)                     \
197 ( {                                                                     \
198     v16u8 res;                                                          \
199     v8i16 res_r, res_l;                                                 \
200     v8u16 sum0_r, sum1_r, sum2_r, sum3_r;                               \
201     v8u16 sum0_l, sum1_l, sum2_l, sum3_l;                               \
202                                                                         \
203     ILVRL_B2_UH(inp4, inp0, sum0_r, sum0_l);                            \
204     ILVRL_B2_UH(inp7, inp3, sum3_r, sum3_l);                            \
205     DOTP_UB2_UH(sum0_r, sum0_l, coef0, coef0, sum0_r, sum0_l);          \
206     HADD_UB2_UH(sum3_r, sum3_l, sum3_r, sum3_l);                        \
207     ILVRL_B2_UH(inp6, inp2, sum2_r, sum2_l);                            \
208     ILVRL_B2_UH(inp5, inp1, sum1_r, sum1_l);                            \
209     DPADD_UB2_UH(sum2_r, sum2_l, coef2, coef2, sum0_r, sum0_l);         \
210     DPADD_UB2_UH(sum1_r, sum1_l, coef1, coef1, sum3_r, sum3_l);         \
211     res_r = (v8i16) (sum0_r - sum3_r);                                  \
212     res_l = (v8i16) (sum0_l - sum3_l);                                  \
213     SRARI_H2_SH(res_r, res_l, 5);                                       \
214     CLIP_SH2_0_255(res_r, res_l);                                       \
215     res = (v16u8) __msa_pckev_b((v16i8) res_l, (v16i8) res_r);          \
216                                                                         \
217     res;                                                                \
218 } )
219
220 #define APPLY_VERT_QPEL_FILTER_8BYTE(inp00, inp01, inp02, inp03,        \
221                                      inp04, inp05, inp06, inp07,        \
222                                      inp10, inp11, inp12, inp13,        \
223                                      inp14, inp15, inp16, inp17,        \
224                                      coef0, coef1, coef2)               \
225 ( {                                                                     \
226     v16u8 res;                                                          \
227     v8i16 val0, val1;                                                   \
228     v8u16 sum00, sum01, sum02, sum03;                                   \
229     v8u16 sum10, sum11, sum12, sum13;                                   \
230                                                                         \
231     ILVR_B4_UH(inp04, inp00, inp14, inp10, inp07, inp03, inp17, inp13,  \
232                sum00, sum10, sum03, sum13);                             \
233     DOTP_UB2_UH(sum00, sum10, coef0, coef0, sum00, sum10);              \
234     HADD_UB2_UH(sum03, sum13, sum03, sum13);                            \
235     ILVR_B4_UH(inp06, inp02, inp16, inp12, inp05, inp01, inp15, inp11,  \
236                sum02, sum12, sum01, sum11);                             \
237     DPADD_UB2_UH(sum02, sum12, coef2, coef2, sum00, sum10);             \
238     DPADD_UB2_UH(sum01, sum11, coef1, coef1, sum03, sum13);             \
239     val0 = (v8i16) (sum00 - sum03);                                     \
240     val1 = (v8i16) (sum10 - sum13);                                     \
241     SRARI_H2_SH(val0, val1, 5);                                         \
242     CLIP_SH2_0_255(val0, val1);                                         \
243     res = (v16u8) __msa_pckev_b((v16i8) val1, (v16i8) val0);            \
244                                                                         \
245     res;                                                                \
246 } )
247
248 #define APPLY_VERT_QPEL_NO_ROUND_FILTER(inp0, inp1, inp2, inp3,         \
249                                         inp4, inp5, inp6, inp7,         \
250                                         coef0, coef1, coef2)            \
251 ( {                                                                     \
252     v16u8 res;                                                          \
253     v8i16 res_r, res_l;                                                 \
254     v8u16 sum0_r, sum1_r, sum2_r, sum3_r;                               \
255     v8u16 sum0_l, sum1_l, sum2_l, sum3_l;                               \
256                                                                         \
257     ILVRL_B2_UH(inp4, inp0, sum0_r, sum0_l);                            \
258     ILVRL_B2_UH(inp7, inp3, sum3_r, sum3_l);                            \
259     DOTP_UB2_UH(sum0_r, sum0_l, coef0, coef0, sum0_r, sum0_l);          \
260     HADD_UB2_UH(sum3_r, sum3_l, sum3_r, sum3_l);                        \
261     ILVRL_B2_UH(inp6, inp2, sum2_r, sum2_l);                            \
262     ILVRL_B2_UH(inp5, inp1, sum1_r, sum1_l);                            \
263     DPADD_UB2_UH(sum2_r, sum2_l, coef2, coef2, sum0_r, sum0_l);         \
264     DPADD_UB2_UH(sum1_r, sum1_l, coef1, coef1, sum3_r, sum3_l);         \
265     res_r = (v8i16) (sum0_r - sum3_r);                                  \
266     res_l = (v8i16) (sum0_l - sum3_l);                                  \
267     res_r += 15;                                                        \
268     res_l += 15;                                                        \
269     res_r >>= 5;                                                        \
270     res_l >>= 5;                                                        \
271     CLIP_SH2_0_255(res_r, res_l);                                       \
272     res = (v16u8) __msa_pckev_b((v16i8) res_l, (v16i8) res_r);          \
273                                                                         \
274     res;                                                                \
275 } )
276
277 #define APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp00, inp01, inp02, inp03,  \
278                                               inp04, inp05, inp06, inp07,  \
279                                               inp10, inp11, inp12, inp13,  \
280                                               inp14, inp15, inp16, inp17,  \
281                                               coef0, coef1, coef2)         \
282 ( {                                                                        \
283     v16u8 res;                                                             \
284     v8i16 val0, val1;                                                      \
285     v8u16 sum00, sum01, sum02, sum03;                                      \
286     v8u16 sum10, sum11, sum12, sum13;                                      \
287                                                                            \
288     ILVR_B4_UH(inp04, inp00, inp14, inp10, inp07, inp03, inp17, inp13,     \
289                sum00, sum10, sum03, sum13);                                \
290     DOTP_UB2_UH(sum00, sum10, coef0, coef0, sum00, sum10);                 \
291     HADD_UB2_UH(sum03, sum13, sum03, sum13);                               \
292     ILVR_B4_UH(inp06, inp02, inp16, inp12, inp05, inp01, inp15, inp11,     \
293                sum02, sum12, sum01, sum11);                                \
294     DPADD_UB2_UH(sum02, sum12, coef2, coef2, sum00, sum10);                \
295     DPADD_UB2_UH(sum01, sum11, coef1, coef1, sum03, sum13);                \
296     val0 = (v8i16) (sum00 - sum03);                                        \
297     val1 = (v8i16) (sum10 - sum13);                                        \
298     val0 += 15;                                                            \
299     val1 += 15;                                                            \
300     val0 >>= 5;                                                            \
301     val1 >>= 5;                                                            \
302     CLIP_SH2_0_255(val0, val1);                                            \
303     res = (v16u8) __msa_pckev_b((v16i8) val1, (v16i8) val0);               \
304                                                                            \
305     res;                                                                   \
306 } )
307
308 static void horiz_mc_qpel_aver_src0_8width_msa(const uint8_t *src,
309                                                int32_t src_stride,
310                                                uint8_t *dst,
311                                                int32_t dst_stride,
312                                                int32_t height)
313 {
314     uint8_t loop_count;
315     v16u8 inp0, inp1, inp2, inp3;
316     v16u8 res0, res1;
317     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
318     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
319     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
320     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
321     v16u8 const20 = (v16u8) __msa_ldi_b(20);
322     v16u8 const6 = (v16u8) __msa_ldi_b(6);
323     v16u8 const3 = (v16u8) __msa_ldi_b(3);
324
325     for (loop_count = (height >> 2); loop_count--;) {
326         LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
327         src += (4 * src_stride);
328         res0 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp0, inp1,
329                                              mask0, mask1, mask2, mask3,
330                                              const20, const6, const3);
331         res1 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp2, inp3,
332                                              mask0, mask1, mask2, mask3,
333                                              const20, const6, const3);
334         inp0 = (v16u8) __msa_insve_d((v2i64) inp0, 1, (v2i64) inp1);
335         inp2 = (v16u8) __msa_insve_d((v2i64) inp2, 1, (v2i64) inp3);
336         AVER_UB2_UB(inp0, res0, inp2, res1, res0, res1);
337         ST8x4_UB(res0, res1, dst, dst_stride);
338         dst += (4 * dst_stride);
339     }
340 }
341
342 static void horiz_mc_qpel_aver_src0_16width_msa(const uint8_t *src,
343                                                 int32_t src_stride,
344                                                 uint8_t *dst,
345                                                 int32_t dst_stride,
346                                                 int32_t height)
347 {
348     uint8_t loop_count;
349     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
350     v16u8 res;
351     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
352     v16u8 const6 = (v16u8) __msa_ldi_b(6);
353     v16u8 const3 = (v16u8) __msa_ldi_b(3);
354     v8u16 const20 = (v8u16) __msa_ldi_h(20);
355
356     for (loop_count = (height >> 2); loop_count--;) {
357         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
358         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
359         src += (4 * src_stride);
360         res = APPLY_HORIZ_QPEL_FILTER(inp0, inp1, mask,
361                                       const20, const6, const3);
362         res = __msa_aver_u_b(inp0, res);
363         ST_UB(res, dst);
364         dst += dst_stride;
365
366         res = APPLY_HORIZ_QPEL_FILTER(inp2, inp3, mask,
367                                       const20, const6, const3);
368         res = __msa_aver_u_b(inp2, res);
369         ST_UB(res, dst);
370         dst += dst_stride;
371
372         res = APPLY_HORIZ_QPEL_FILTER(inp4, inp5, mask,
373                                       const20, const6, const3);
374         res = __msa_aver_u_b(inp4, res);
375         ST_UB(res, dst);
376         dst += dst_stride;
377
378         res = APPLY_HORIZ_QPEL_FILTER(inp6, inp7, mask,
379                                       const20, const6, const3);
380         res = __msa_aver_u_b(inp6, res);
381         ST_UB(res, dst);
382         dst += dst_stride;
383     }
384 }
385
386 static void horiz_mc_qpel_8width_msa(const uint8_t *src,
387                                      int32_t src_stride,
388                                      uint8_t *dst,
389                                      int32_t dst_stride,
390                                      int32_t height)
391 {
392     uint8_t loop_count;
393     v16u8 inp0, inp1, inp2, inp3;
394     v16u8 res0, res1;
395     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
396     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
397     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
398     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
399     v16u8 const20 = (v16u8) __msa_ldi_b(20);
400     v16u8 const6 = (v16u8) __msa_ldi_b(6);
401     v16u8 const3 = (v16u8) __msa_ldi_b(3);
402
403     for (loop_count = (height >> 2); loop_count--;) {
404         LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
405         src += (4 * src_stride);
406         res0 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp0, inp1,
407                                              mask0, mask1, mask2, mask3,
408                                              const20, const6, const3);
409         res1 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp2, inp3,
410                                              mask0, mask1, mask2, mask3,
411                                              const20, const6, const3);
412         ST8x4_UB(res0, res1, dst, dst_stride);
413         dst += (4 * dst_stride);
414     }
415 }
416
417 static void horiz_mc_qpel_16width_msa(const uint8_t *src,
418                                       int32_t src_stride,
419                                       uint8_t *dst,
420                                       int32_t dst_stride,
421                                       int32_t height)
422 {
423     uint8_t loop_count;
424     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
425     v16u8 res;
426     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
427     v8u16 const20 = (v8u16) __msa_ldi_h(20);
428     v16u8 const6 = (v16u8) __msa_ldi_b(6);
429     v16u8 const3 = (v16u8) __msa_ldi_b(3);
430
431     for (loop_count = (height >> 2); loop_count--;) {
432         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
433         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
434         src += (4 * src_stride);
435         res = APPLY_HORIZ_QPEL_FILTER(inp0, inp1, mask,
436                                       const20, const6, const3);
437         ST_UB(res, dst);
438         dst += dst_stride;
439
440         res = APPLY_HORIZ_QPEL_FILTER(inp2, inp3, mask,
441                                       const20, const6, const3);
442         ST_UB(res, dst);
443         dst += dst_stride;
444
445         res = APPLY_HORIZ_QPEL_FILTER(inp4, inp5, mask,
446                                       const20, const6, const3);
447         ST_UB(res, dst);
448         dst += dst_stride;
449
450         res = APPLY_HORIZ_QPEL_FILTER(inp6, inp7, mask,
451                                       const20, const6, const3);
452         ST_UB(res, dst);
453         dst += dst_stride;
454     }
455 }
456
457 static void horiz_mc_qpel_aver_src1_8width_msa(const uint8_t *src,
458                                                int32_t src_stride,
459                                                uint8_t *dst,
460                                                int32_t dst_stride,
461                                                int32_t height)
462 {
463     uint8_t loop_count;
464     v16u8 inp0, inp1, inp2, inp3;
465     v16u8 res0, res1;
466     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
467     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
468     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
469     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
470     v16u8 const20 = (v16u8) __msa_ldi_b(20);
471     v16u8 const6 = (v16u8) __msa_ldi_b(6);
472     v16u8 const3 = (v16u8) __msa_ldi_b(3);
473
474     for (loop_count = (height >> 2); loop_count--;) {
475         LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
476         src += (4 * src_stride);
477         res0 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp0, inp1,
478                                              mask0, mask1, mask2, mask3,
479                                              const20, const6, const3);
480         res1 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp2, inp3,
481                                              mask0, mask1, mask2, mask3,
482                                              const20, const6, const3);
483         SLDI_B2_UB(inp0, inp1, inp0, inp1, inp0, inp1, 1);
484         SLDI_B2_UB(inp2, inp3, inp2, inp3, inp2, inp3, 1);
485         inp0 = (v16u8) __msa_insve_d((v2i64) inp0, 1, (v2i64) inp1);
486         inp2 = (v16u8) __msa_insve_d((v2i64) inp2, 1, (v2i64) inp3);
487         AVER_UB2_UB(inp0, res0, inp2, res1, res0, res1);
488         ST8x4_UB(res0, res1, dst, dst_stride);
489         dst += (4 * dst_stride);
490     }
491 }
492
493 static void horiz_mc_qpel_aver_src1_16width_msa(const uint8_t *src,
494                                                 int32_t src_stride,
495                                                 uint8_t *dst,
496                                                 int32_t dst_stride,
497                                                 int32_t height)
498 {
499     uint8_t loop_count;
500     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
501     v16u8 res;
502     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
503     v8u16 const20 = (v8u16) __msa_ldi_h(20);
504     v16u8 const6 = (v16u8) __msa_ldi_b(6);
505     v16u8 const3 = (v16u8) __msa_ldi_b(3);
506
507     for (loop_count = (height >> 2); loop_count--;) {
508         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
509         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
510         src += (4 * src_stride);
511         res = APPLY_HORIZ_QPEL_FILTER(inp0, inp1, mask,
512                                       const20, const6, const3);
513         res = __msa_aver_u_b(res, inp1);
514         ST_UB(res, dst);
515         dst += dst_stride;
516
517         res = APPLY_HORIZ_QPEL_FILTER(inp2, inp3, mask,
518                                       const20, const6, const3);
519         res = __msa_aver_u_b(res, inp3);
520         ST_UB(res, dst);
521         dst += dst_stride;
522
523         res = APPLY_HORIZ_QPEL_FILTER(inp4, inp5, mask,
524                                       const20, const6, const3);
525         res = __msa_aver_u_b(res, inp5);
526         ST_UB(res, dst);
527         dst += dst_stride;
528
529         res = APPLY_HORIZ_QPEL_FILTER(inp6, inp7, mask,
530                                       const20, const6, const3);
531         res = __msa_aver_u_b(res, inp7);
532         ST_UB(res, dst);
533         dst += dst_stride;
534     }
535 }
536
537 static void horiz_mc_qpel_no_rnd_aver_src0_8width_msa(const uint8_t *src,
538                                                       int32_t src_stride,
539                                                       uint8_t *dst,
540                                                       int32_t dst_stride,
541                                                       int32_t height)
542 {
543     uint8_t loop_count;
544     v16u8 inp0, inp1, inp2, inp3;
545     v16u8 res0, res1;
546     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
547     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
548     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
549     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
550     v16u8 const20 = (v16u8) __msa_ldi_b(20);
551     v16u8 const6 = (v16u8) __msa_ldi_b(6);
552     v16u8 const3 = (v16u8) __msa_ldi_b(3);
553
554     for (loop_count = (height >> 2); loop_count--;) {
555         LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
556         src += (4 * src_stride);
557         res0 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp0, inp1, mask0, mask1,
558                                                       mask2, mask3, const20,
559                                                       const6, const3);
560         res1 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp2, inp3, mask0, mask1,
561                                                       mask2, mask3, const20,
562                                                       const6, const3);
563         inp0 = (v16u8) __msa_insve_d((v2i64) inp0, 1, (v2i64) inp1);
564         inp2 = (v16u8) __msa_insve_d((v2i64) inp2, 1, (v2i64) inp3);
565         res0 = __msa_ave_u_b(inp0, res0);
566         res1 = __msa_ave_u_b(inp2, res1);
567         ST8x4_UB(res0, res1, dst, dst_stride);
568         dst += (4 * dst_stride);
569     }
570 }
571
572 static void horiz_mc_qpel_no_rnd_aver_src0_16width_msa(const uint8_t *src,
573                                                        int32_t src_stride,
574                                                        uint8_t *dst,
575                                                        int32_t dst_stride,
576                                                        int32_t height)
577 {
578     uint8_t loop_count;
579     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
580     v16u8 res;
581     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
582     v8u16 const20 = (v8u16) __msa_ldi_h(20);
583     v16u8 const6 = (v16u8) __msa_ldi_b(6);
584     v16u8 const3 = (v16u8) __msa_ldi_b(3);
585
586     for (loop_count = (height >> 2); loop_count--;) {
587         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
588         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
589         src += (4 * src_stride);
590         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp0, inp1, mask,
591                                                const20, const6, const3);
592         res = __msa_ave_u_b(inp0, res);
593         ST_UB(res, dst);
594         dst += dst_stride;
595
596         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp2, inp3, mask,
597                                                const20, const6, const3);
598         res = __msa_ave_u_b(inp2, res);
599         ST_UB(res, dst);
600         dst += dst_stride;
601
602         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp4, inp5, mask,
603                                                const20, const6, const3);
604         res = __msa_ave_u_b(inp4, res);
605         ST_UB(res, dst);
606         dst += dst_stride;
607
608         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp6, inp7, mask,
609                                                const20, const6, const3);
610         res = __msa_ave_u_b(inp6, res);
611         ST_UB(res, dst);
612         dst += dst_stride;
613     }
614 }
615
616 static void horiz_mc_qpel_no_rnd_8width_msa(const uint8_t *src,
617                                             int32_t src_stride,
618                                             uint8_t *dst,
619                                             int32_t dst_stride,
620                                             int32_t height)
621 {
622     uint8_t loop_count;
623     v16u8 inp0, inp1, inp2, inp3;
624     v16u8 res0, res1;
625     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
626     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
627     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
628     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
629     v16u8 const20 = (v16u8) __msa_ldi_b(20);
630     v16u8 const6 = (v16u8) __msa_ldi_b(6);
631     v16u8 const3 = (v16u8) __msa_ldi_b(3);
632
633     for (loop_count = (height >> 2); loop_count--;) {
634         LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
635         src += (4 * src_stride);
636         res0 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp0, inp1, mask0, mask1,
637                                                       mask2, mask3, const20,
638                                                       const6, const3);
639         res1 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp2, inp3, mask0, mask1,
640                                                       mask2, mask3, const20,
641                                                       const6, const3);
642         ST8x4_UB(res0, res1, dst, dst_stride);
643         dst += (4 * dst_stride);
644     }
645 }
646
647 static void horiz_mc_qpel_no_rnd_16width_msa(const uint8_t *src,
648                                              int32_t src_stride,
649                                              uint8_t *dst,
650                                              int32_t dst_stride,
651                                              int32_t height)
652 {
653     uint8_t loop_count;
654     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
655     v16u8 res;
656     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
657     v16u8 const6 = (v16u8) __msa_ldi_b(6);
658     v16u8 const3 = (v16u8) __msa_ldi_b(3);
659     v8u16 const20 = (v8u16) __msa_ldi_h(20);
660
661     for (loop_count = (height >> 2); loop_count--;) {
662         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
663         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
664         src += (4 * src_stride);
665         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp0, inp1, mask,
666                                                const20, const6, const3);
667         ST_UB(res, dst);
668         dst += dst_stride;
669
670         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp2, inp3, mask,
671                                                const20, const6, const3);
672         ST_UB(res, dst);
673         dst += dst_stride;
674
675         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp4, inp5, mask,
676                                                const20, const6, const3);
677         ST_UB(res, dst);
678         dst += dst_stride;
679
680         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp6, inp7, mask,
681                                                const20, const6, const3);
682         ST_UB(res, dst);
683         dst += dst_stride;
684     }
685 }
686
687 static void horiz_mc_qpel_no_rnd_aver_src1_8width_msa(const uint8_t *src,
688                                                       int32_t src_stride,
689                                                       uint8_t *dst,
690                                                       int32_t dst_stride,
691                                                       int32_t height)
692 {
693     uint8_t loop_count;
694     v16u8 inp0, inp1, inp2, inp3;
695     v16u8 res0, res1;
696     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
697     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
698     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
699     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
700     v16u8 const20 = (v16u8) __msa_ldi_b(20);
701     v16u8 const6 = (v16u8) __msa_ldi_b(6);
702     v16u8 const3 = (v16u8) __msa_ldi_b(3);
703
704     for (loop_count = (height >> 2); loop_count--;) {
705         LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
706         src += (4 * src_stride);
707         res0 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp0, inp1, mask0, mask1,
708                                                       mask2, mask3, const20,
709                                                       const6, const3);
710         res1 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp2, inp3, mask0, mask1,
711                                                       mask2, mask3, const20,
712                                                       const6, const3);
713         SLDI_B2_UB(inp0, inp1, inp0, inp1, inp0, inp1, 1);
714         SLDI_B2_UB(inp2, inp3, inp2, inp3, inp2, inp3, 1);
715         inp0 = (v16u8) __msa_insve_d((v2i64) inp0, 1, (v2i64) inp1);
716         inp2 = (v16u8) __msa_insve_d((v2i64) inp2, 1, (v2i64) inp3);
717         res0 = __msa_ave_u_b(inp0, res0);
718         res1 = __msa_ave_u_b(inp2, res1);
719         ST8x4_UB(res0, res1, dst, dst_stride);
720         dst += (4 * dst_stride);
721     }
722 }
723
724 static void horiz_mc_qpel_no_rnd_aver_src1_16width_msa(const uint8_t *src,
725                                                        int32_t src_stride,
726                                                        uint8_t *dst,
727                                                        int32_t dst_stride,
728                                                        int32_t height)
729 {
730     uint8_t loop_count;
731     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
732     v16u8 res;
733     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
734     v16u8 const6 = (v16u8) __msa_ldi_b(6);
735     v16u8 const3 = (v16u8) __msa_ldi_b(3);
736     v8u16 const20 = (v8u16) __msa_ldi_h(20);
737
738     for (loop_count = (height >> 2); loop_count--;) {
739         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
740         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
741         src += (4 * src_stride);
742         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp0, inp1, mask,
743                                                const20, const6, const3);
744         res = __msa_ave_u_b(res, inp1);
745         ST_UB(res, dst);
746         dst += dst_stride;
747
748         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp2, inp3, mask,
749                                                const20, const6, const3);
750         res = __msa_ave_u_b(res, inp3);
751         ST_UB(res, dst);
752         dst += dst_stride;
753
754         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp4, inp5, mask,
755                                                const20, const6, const3);
756         res = __msa_ave_u_b(res, inp5);
757         ST_UB(res, dst);
758         dst += dst_stride;
759
760         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp6, inp7, mask,
761                                                const20, const6, const3);
762         res = __msa_ave_u_b(res, inp7);
763         ST_UB(res, dst);
764         dst += dst_stride;
765     }
766 }
767
768 static void horiz_mc_qpel_avg_dst_aver_src0_8width_msa(const uint8_t *src,
769                                                        int32_t src_stride,
770                                                        uint8_t *dst,
771                                                        int32_t dst_stride,
772                                                        int32_t height)
773 {
774     uint8_t loop_count;
775     v16u8 inp0, inp1, inp2, inp3;
776     v16u8 dst0, dst1, dst2, dst3;
777     v16u8 res0, res1;
778     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
779     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
780     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
781     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
782     v16u8 const20 = (v16u8) __msa_ldi_b(20);
783     v16u8 const6 = (v16u8) __msa_ldi_b(6);
784     v16u8 const3 = (v16u8) __msa_ldi_b(3);
785
786     for (loop_count = (height >> 2); loop_count--;) {
787         LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
788         src += (4 * src_stride);
789         res0 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp0, inp1,
790                                              mask0, mask1, mask2, mask3,
791                                              const20, const6, const3);
792         res1 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp2, inp3,
793                                              mask0, mask1, mask2, mask3,
794                                              const20, const6, const3);
795         LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
796         inp0 = (v16u8) __msa_insve_d((v2i64) inp0, 1, (v2i64) inp1);
797         inp2 = (v16u8) __msa_insve_d((v2i64) inp2, 1, (v2i64) inp3);
798         dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
799         dst2 = (v16u8) __msa_insve_d((v2i64) dst2, 1, (v2i64) dst3);
800         AVER_UB2_UB(inp0, res0, inp2, res1, res0, res1);
801         AVER_UB2_UB(dst0, res0, dst2, res1, res0, res1);
802         ST8x4_UB(res0, res1, dst, dst_stride);
803         dst += (4 * dst_stride);
804     }
805 }
806
807 static void horiz_mc_qpel_avg_dst_aver_src0_16width_msa(const uint8_t *src,
808                                                         int32_t src_stride,
809                                                         uint8_t *dst,
810                                                         int32_t dst_stride,
811                                                         int32_t height)
812 {
813     uint8_t loop_count;
814     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
815     v16u8 res0, res1;
816     v16u8 dst0, dst1;
817     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
818     v16u8 const6 = (v16u8) __msa_ldi_b(6);
819     v16u8 const3 = (v16u8) __msa_ldi_b(3);
820     v8u16 const20 = (v8u16) __msa_ldi_h(20);
821
822     for (loop_count = (height >> 2); loop_count--;) {
823         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
824         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
825         src += (4 * src_stride);
826         res0 = APPLY_HORIZ_QPEL_FILTER(inp0, inp1, mask,
827                                        const20, const6, const3);
828         res1 = APPLY_HORIZ_QPEL_FILTER(inp2, inp3, mask,
829                                        const20, const6, const3);
830         LD_UB2(dst, dst_stride, dst0, dst1);
831         AVER_UB2_UB(inp0, res0, inp2, res1, res0, res1);
832         AVER_UB2_UB(dst0, res0, dst1, res1, res0, res1);
833         ST_UB2(res0, res1, dst, dst_stride);
834         dst += (2 * dst_stride);
835
836         res0 = APPLY_HORIZ_QPEL_FILTER(inp4, inp5, mask,
837                                        const20, const6, const3);
838         res1 = APPLY_HORIZ_QPEL_FILTER(inp6, inp7, mask,
839                                        const20, const6, const3);
840         LD_UB2(dst, dst_stride, dst0, dst1);
841         AVER_UB2_UB(inp4, res0, inp6, res1, res0, res1);
842         AVER_UB2_UB(dst0, res0, dst1, res1, res0, res1);
843         ST_UB2(res0, res1, dst, dst_stride);
844         dst += (2 * dst_stride);
845     }
846 }
847
848 static void horiz_mc_qpel_avg_dst_8width_msa(const uint8_t *src,
849                                              int32_t src_stride,
850                                              uint8_t *dst,
851                                              int32_t dst_stride,
852                                              int32_t height)
853 {
854     uint8_t loop_count;
855     v16u8 inp0, inp1, inp2, inp3;
856     v16u8 dst0, dst1, dst2, dst3;
857     v16u8 res0, res1;
858     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
859     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
860     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
861     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
862     v16u8 const20 = (v16u8) __msa_ldi_b(20);
863     v16u8 const6 = (v16u8) __msa_ldi_b(6);
864     v16u8 const3 = (v16u8) __msa_ldi_b(3);
865
866     for (loop_count = (height >> 2); loop_count--;) {
867         LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
868         src += (4 * src_stride);
869         res0 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp0, inp1,
870                                              mask0, mask1, mask2, mask3,
871                                              const20, const6, const3);
872         res1 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp2, inp3,
873                                              mask0, mask1, mask2, mask3,
874                                              const20, const6, const3);
875         LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
876         dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
877         dst2 = (v16u8) __msa_insve_d((v2i64) dst2, 1, (v2i64) dst3);
878         AVER_UB2_UB(dst0, res0, dst2, res1, res0, res1);
879         ST8x4_UB(res0, res1, dst, dst_stride);
880         dst += (4 * dst_stride);
881     }
882 }
883
884 static void horiz_mc_qpel_avg_dst_16width_msa(const uint8_t *src,
885                                               int32_t src_stride,
886                                               uint8_t *dst,
887                                               int32_t dst_stride,
888                                               int32_t height)
889 {
890     uint8_t loop_count;
891     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
892     v16u8 res0, res1;
893     v16u8 dst0, dst1;
894     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
895     v16u8 const6 = (v16u8) __msa_ldi_b(6);
896     v16u8 const3 = (v16u8) __msa_ldi_b(3);
897     v8u16 const20 = (v8u16) __msa_ldi_h(20);
898
899     for (loop_count = (height >> 2); loop_count--;) {
900         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
901         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
902         src += (4 * src_stride);
903         res0 = APPLY_HORIZ_QPEL_FILTER(inp0, inp1, mask,
904                                        const20, const6, const3);
905         res1 = APPLY_HORIZ_QPEL_FILTER(inp2, inp3, mask,
906                                        const20, const6, const3);
907         LD_UB2(dst, dst_stride, dst0, dst1);
908         AVER_UB2_UB(dst0, res0, dst1, res1, res0, res1);
909         ST_UB2(res0, res1, dst, dst_stride);
910         dst += (2 * dst_stride);
911
912         res0 = APPLY_HORIZ_QPEL_FILTER(inp4, inp5, mask,
913                                        const20, const6, const3);
914         res1 = APPLY_HORIZ_QPEL_FILTER(inp6, inp7, mask,
915                                        const20, const6, const3);
916         LD_UB2(dst, dst_stride, dst0, dst1);
917         AVER_UB2_UB(dst0, res0, dst1, res1, res0, res1);
918         ST_UB2(res0, res1, dst, dst_stride);
919         dst += (2 * dst_stride);
920     }
921 }
922
923 static void horiz_mc_qpel_avg_dst_aver_src1_8width_msa(const uint8_t *src,
924                                                        int32_t src_stride,
925                                                        uint8_t *dst,
926                                                        int32_t dst_stride,
927                                                        int32_t height)
928 {
929     uint8_t loop_count;
930     v16u8 inp0, inp1, inp2, inp3;
931     v16u8 dst0, dst1, dst2, dst3;
932     v16u8 res0, res1;
933     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
934     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
935     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
936     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
937     v16u8 const20 = (v16u8) __msa_ldi_b(20);
938     v16u8 const6 = (v16u8) __msa_ldi_b(6);
939     v16u8 const3 = (v16u8) __msa_ldi_b(3);
940
941     for (loop_count = (height >> 2); loop_count--;) {
942         LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
943         src += (4 * src_stride);
944         res0 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp0, inp1,
945                                              mask0, mask1, mask2, mask3,
946                                              const20, const6, const3);
947         res1 = APPLY_HORIZ_QPEL_FILTER_8BYTE(inp2, inp3,
948                                              mask0, mask1, mask2, mask3,
949                                              const20, const6, const3);
950         LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
951         SLDI_B2_UB(inp0, inp1, inp0, inp1, inp0, inp1, 1);
952         SLDI_B2_UB(inp2, inp3, inp2, inp3, inp2, inp3, 1);
953         inp0 = (v16u8) __msa_insve_d((v2i64) inp0, 1, (v2i64) inp1);
954         inp2 = (v16u8) __msa_insve_d((v2i64) inp2, 1, (v2i64) inp3);
955         dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
956         dst2 = (v16u8) __msa_insve_d((v2i64) dst2, 1, (v2i64) dst3);
957         AVER_UB2_UB(inp0, res0, inp2, res1, res0, res1);
958         AVER_UB2_UB(dst0, res0, dst2, res1, res0, res1);
959         ST8x4_UB(res0, res1, dst, dst_stride);
960         dst += (4 * dst_stride);
961     }
962 }
963
964 static void horiz_mc_qpel_avg_dst_aver_src1_16width_msa(const uint8_t *src,
965                                                         int32_t src_stride,
966                                                         uint8_t *dst,
967                                                         int32_t dst_stride,
968                                                         int32_t height)
969 {
970     uint8_t loop_count;
971     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
972     v16u8 res0, res1, dst0, dst1;
973     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
974     v16u8 const6 = (v16u8) __msa_ldi_b(6);
975     v16u8 const3 = (v16u8) __msa_ldi_b(3);
976     v8u16 const20 = (v8u16) __msa_ldi_h(20);
977
978     for (loop_count = (height >> 2); loop_count--;) {
979         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
980         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
981         src += (4 * src_stride);
982         res0 = APPLY_HORIZ_QPEL_FILTER(inp0, inp1, mask,
983                                        const20, const6, const3);
984         res1 = APPLY_HORIZ_QPEL_FILTER(inp2, inp3, mask,
985                                        const20, const6, const3);
986         LD_UB2(dst, dst_stride, dst0, dst1);
987         AVER_UB2_UB(res0, inp1, res1, inp3, res0, res1);
988         AVER_UB2_UB(dst0, res0, dst1, res1, res0, res1);
989         ST_UB2(res0, res1, dst, dst_stride);
990         dst += (2 * dst_stride);
991         res0 = APPLY_HORIZ_QPEL_FILTER(inp4, inp5, mask,
992                                        const20, const6, const3);
993         res1 = APPLY_HORIZ_QPEL_FILTER(inp6, inp7, mask,
994                                        const20, const6, const3);
995         LD_UB2(dst, dst_stride, dst0, dst1);
996         AVER_UB2_UB(res0, inp5, res1, inp7, res0, res1);
997         AVER_UB2_UB(dst0, res0, dst1, res1, res0, res1);
998         ST_UB2(res0, res1, dst, dst_stride);
999         dst += (2 * dst_stride);
1000     }
1001 }
1002
1003
1004 static void vert_mc_qpel_aver_src0_8x8_msa(const uint8_t *src,
1005                                            int32_t src_stride,
1006                                            uint8_t *dst,
1007                                            int32_t dst_stride)
1008 {
1009     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1010     v16u8 tmp0, tmp1, res0, res1;
1011     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1012     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1013     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1014
1015     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
1016     src += (4 * src_stride);
1017     LD_UB2(src, src_stride, inp4, inp5);
1018     src += (2 * src_stride);
1019     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp0, inp0, inp1, inp2,
1020                                         inp1, inp2, inp3, inp4,
1021                                         inp1, inp0, inp0, inp1,
1022                                         inp2, inp3, inp4, inp5,
1023                                         const20, const6, const3);
1024     LD_UB2(src, src_stride, inp6, inp7);
1025     src += (2 * src_stride);
1026     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp2, inp1, inp0, inp0,
1027                                         inp3, inp4, inp5, inp6,
1028                                         inp3, inp2, inp1, inp0,
1029                                         inp4, inp5, inp6, inp7,
1030                                         const20, const6, const3);
1031     tmp0 = (v16u8) __msa_insve_d((v2i64) inp0, 1, (v2i64) inp1);
1032     tmp1 = (v16u8) __msa_insve_d((v2i64) inp2, 1, (v2i64) inp3);
1033     AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
1034     ST8x4_UB(res0, res1, dst, dst_stride);
1035     dst += (4 * dst_stride);
1036
1037     inp8 = LD_UB(src);
1038     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp4, inp3, inp2, inp1,
1039                                         inp5, inp6, inp7, inp8,
1040                                         inp5, inp4, inp3, inp2,
1041                                         inp6, inp7, inp8, inp8,
1042                                         const20, const6, const3);
1043     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp6, inp5, inp4, inp3,
1044                                         inp7, inp8, inp8, inp7,
1045                                         inp7, inp6, inp5, inp4,
1046                                         inp8, inp8, inp7, inp6,
1047                                         const20, const6, const3);
1048     tmp0 = (v16u8) __msa_insve_d((v2i64) inp4, 1, (v2i64) inp5);
1049     tmp1 = (v16u8) __msa_insve_d((v2i64) inp6, 1, (v2i64) inp7);
1050     AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
1051     ST8x4_UB(res0, res1, dst, dst_stride);
1052     dst += (4 * dst_stride);
1053 }
1054
1055 static void vert_mc_qpel_aver_src0_16x16_msa(const uint8_t *src,
1056                                              int32_t src_stride,
1057                                              uint8_t *dst,
1058                                              int32_t dst_stride)
1059 {
1060     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1061     v16u8 inp9, inp10, inp11, inp12, inp13, inp14, inp15, inp16;
1062     v16u8 res0;
1063     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1064     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1065     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1066
1067     LD_UB5(src, src_stride, inp0, inp1, inp2, inp3, inp4);
1068     src += (5 * src_stride);
1069     res0 = APPLY_VERT_QPEL_FILTER(inp0, inp0, inp1, inp2,
1070                                   inp1, inp2, inp3, inp4,
1071                                   const20, const6, const3);
1072     res0 = __msa_aver_u_b(res0, inp0);
1073     ST_UB(res0, dst);
1074     dst += dst_stride;
1075
1076     inp5 = LD_UB(src);
1077     src += src_stride;
1078     res0 = APPLY_VERT_QPEL_FILTER(inp1, inp0, inp0, inp1,
1079                                   inp2, inp3, inp4, inp5,
1080                                   const20, const6, const3);
1081     res0 = __msa_aver_u_b(res0, inp1);
1082     ST_UB(res0, dst);
1083     dst += dst_stride;
1084
1085     inp6 = LD_UB(src);
1086     src += src_stride;
1087     res0 = APPLY_VERT_QPEL_FILTER(inp2, inp1, inp0, inp0,
1088                                   inp3, inp4, inp5, inp6,
1089                                   const20, const6, const3);
1090     res0 = __msa_aver_u_b(res0, inp2);
1091     ST_UB(res0, dst);
1092     dst += dst_stride;
1093
1094     inp7 = LD_UB(src);
1095     src += src_stride;
1096     res0 = APPLY_VERT_QPEL_FILTER(inp3, inp2, inp1, inp0,
1097                                   inp4, inp5, inp6, inp7,
1098                                   const20, const6, const3);
1099     res0 = __msa_aver_u_b(res0, inp3);
1100     ST_UB(res0, dst);
1101     dst += dst_stride;
1102
1103     LD_UB2(src, src_stride, inp8, inp9);
1104     src += (2 * src_stride);
1105     res0 = APPLY_VERT_QPEL_FILTER(inp4, inp3, inp2, inp1,
1106                                   inp5, inp6, inp7, inp8,
1107                                   const20, const6, const3);
1108     res0 = __msa_aver_u_b(res0, inp4);
1109     ST_UB(res0, dst);
1110     dst += dst_stride;
1111
1112     res0 = APPLY_VERT_QPEL_FILTER(inp5, inp4, inp3, inp2,
1113                                   inp6, inp7, inp8, inp9,
1114                                   const20, const6, const3);
1115     res0 = __msa_aver_u_b(res0, inp5);
1116     ST_UB(res0, dst);
1117     dst += dst_stride;
1118
1119     LD_UB2(src, src_stride, inp10, inp11);
1120     src += (2 * src_stride);
1121     res0 = APPLY_VERT_QPEL_FILTER(inp6, inp5, inp4, inp3,
1122                                   inp7, inp8, inp9, inp10,
1123                                   const20, const6, const3);
1124     res0 = __msa_aver_u_b(res0, inp6);
1125     ST_UB(res0, dst);
1126     dst += dst_stride;
1127
1128     res0 = APPLY_VERT_QPEL_FILTER(inp7, inp6, inp5, inp4,
1129                                   inp8, inp9, inp10, inp11,
1130                                   const20, const6, const3);
1131     res0 = __msa_aver_u_b(res0, inp7);
1132     ST_UB(res0, dst);
1133     dst += dst_stride;
1134
1135     LD_UB2(src, src_stride, inp12, inp13);
1136     src += (2 * src_stride);
1137     res0 = APPLY_VERT_QPEL_FILTER(inp8, inp7, inp6, inp5,
1138                                   inp9, inp10, inp11, inp12,
1139                                   const20, const6, const3);
1140     res0 = __msa_aver_u_b(res0, inp8);
1141     ST_UB(res0, dst);
1142     dst += dst_stride;
1143
1144     res0 = APPLY_VERT_QPEL_FILTER(inp9, inp8, inp7, inp6,
1145                                   inp10, inp11, inp12, inp13,
1146                                   const20, const6, const3);
1147     res0 = __msa_aver_u_b(res0, inp9);
1148     ST_UB(res0, dst);
1149     dst += dst_stride;
1150
1151     LD_UB2(src, src_stride, inp14, inp15);
1152     src += (2 * src_stride);
1153     res0 = APPLY_VERT_QPEL_FILTER(inp10, inp9, inp8, inp7,
1154                                   inp11, inp12, inp13, inp14,
1155                                   const20, const6, const3);
1156     res0 = __msa_aver_u_b(res0, inp10);
1157     ST_UB(res0, dst);
1158     dst += dst_stride;
1159
1160     res0 = APPLY_VERT_QPEL_FILTER(inp11, inp10, inp9, inp8,
1161                                   inp12, inp13, inp14, inp15,
1162                                   const20, const6, const3);
1163     res0 = __msa_aver_u_b(res0, inp11);
1164     ST_UB(res0, dst);
1165     dst += dst_stride;
1166
1167     inp16 = LD_UB(src);
1168     res0 = APPLY_VERT_QPEL_FILTER(inp12, inp11, inp10, inp9,
1169                                   inp13, inp14, inp15, inp16,
1170                                   const20, const6, const3);
1171     res0 = __msa_aver_u_b(res0, inp12);
1172     ST_UB(res0, dst);
1173     dst += dst_stride;
1174
1175     res0 = APPLY_VERT_QPEL_FILTER(inp13, inp12, inp11, inp10,
1176                                   inp14, inp15, inp16, inp16,
1177                                   const20, const6, const3);
1178     res0 = __msa_aver_u_b(res0, inp13);
1179     ST_UB(res0, dst);
1180     dst += dst_stride;
1181
1182     res0 = APPLY_VERT_QPEL_FILTER(inp14, inp13, inp12, inp11,
1183                                   inp15, inp16, inp16, inp15,
1184                                   const20, const6, const3);
1185     res0 = __msa_aver_u_b(res0, inp14);
1186     ST_UB(res0, dst);
1187     dst += dst_stride;
1188
1189     res0 = APPLY_VERT_QPEL_FILTER(inp15, inp14, inp13, inp12,
1190                                   inp16, inp16, inp15, inp14,
1191                                   const20, const6, const3);
1192     res0 = __msa_aver_u_b(res0, inp15);
1193     ST_UB(res0, dst);
1194 }
1195
1196 static void vert_mc_qpel_8x8_msa(const uint8_t *src,
1197                                  int32_t src_stride,
1198                                  uint8_t *dst,
1199                                  int32_t dst_stride)
1200 {
1201     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1202     v16u8 res0, res1;
1203     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1204     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1205     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1206
1207     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
1208     src += (4 * src_stride);
1209     LD_UB2(src, src_stride, inp4, inp5);
1210     src += (2 * src_stride);
1211     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp0, inp0, inp1, inp2,
1212                                         inp1, inp2, inp3, inp4,
1213                                         inp1, inp0, inp0, inp1,
1214                                         inp2, inp3, inp4, inp5,
1215                                         const20, const6, const3);
1216     LD_UB2(src, src_stride, inp6, inp7);
1217     src += (2 * src_stride);
1218     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp2, inp1, inp0, inp0,
1219                                         inp3, inp4, inp5, inp6,
1220                                         inp3, inp2, inp1, inp0,
1221                                         inp4, inp5, inp6, inp7,
1222                                         const20, const6, const3);
1223     ST8x4_UB(res0, res1, dst, dst_stride);
1224     dst += (4 * dst_stride);
1225
1226     inp8 = LD_UB(src);
1227     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp4, inp3, inp2, inp1,
1228                                         inp5, inp6, inp7, inp8,
1229                                         inp5, inp4, inp3, inp2,
1230                                         inp6, inp7, inp8, inp8,
1231                                         const20, const6, const3);
1232     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp6, inp5, inp4, inp3,
1233                                         inp7, inp8, inp8, inp7,
1234                                         inp7, inp6, inp5, inp4,
1235                                         inp8, inp8, inp7, inp6,
1236                                         const20, const6, const3);
1237     ST8x4_UB(res0, res1, dst, dst_stride);
1238     dst += (4 * dst_stride);
1239 }
1240
1241 static void vert_mc_qpel_16x16_msa(const uint8_t *src,
1242                                    int32_t src_stride,
1243                                    uint8_t *dst,
1244                                    int32_t dst_stride)
1245 {
1246     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1247     v16u8 inp9, inp10, inp11, inp12, inp13, inp14, inp15, inp16;
1248     v16u8 res0;
1249     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1250     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1251     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1252
1253     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
1254     src += (4 * src_stride);
1255     inp4 = LD_UB(src);
1256     src += src_stride;
1257     res0 = APPLY_VERT_QPEL_FILTER(inp0, inp0, inp1, inp2,
1258                                   inp1, inp2, inp3, inp4,
1259                                   const20, const6, const3);
1260     ST_UB(res0, dst);
1261     dst += dst_stride;
1262
1263     inp5 = LD_UB(src);
1264     src += src_stride;
1265     res0 = APPLY_VERT_QPEL_FILTER(inp1, inp0, inp0, inp1,
1266                                   inp2, inp3, inp4, inp5,
1267                                   const20, const6, const3);
1268     ST_UB(res0, dst);
1269     dst += dst_stride;
1270
1271     inp6 = LD_UB(src);
1272     src += src_stride;
1273     res0 = APPLY_VERT_QPEL_FILTER(inp2, inp1, inp0, inp0,
1274                                   inp3, inp4, inp5, inp6,
1275                                   const20, const6, const3);
1276     ST_UB(res0, dst);
1277     dst += dst_stride;
1278
1279     inp7 = LD_UB(src);
1280     src += src_stride;
1281     res0 = APPLY_VERT_QPEL_FILTER(inp3, inp2, inp1, inp0,
1282                                   inp4, inp5, inp6, inp7,
1283                                   const20, const6, const3);
1284     ST_UB(res0, dst);
1285     dst += dst_stride;
1286
1287     inp8 = LD_UB(src);
1288     src += src_stride;
1289     res0 = APPLY_VERT_QPEL_FILTER(inp4, inp3, inp2, inp1,
1290                                   inp5, inp6, inp7, inp8,
1291                                   const20, const6, const3);
1292     ST_UB(res0, dst);
1293     dst += dst_stride;
1294
1295     inp9 = LD_UB(src);
1296     src += src_stride;
1297     res0 = APPLY_VERT_QPEL_FILTER(inp5, inp4, inp3, inp2,
1298                                   inp6, inp7, inp8, inp9,
1299                                   const20, const6, const3);
1300     ST_UB(res0, dst);
1301     dst += dst_stride;
1302
1303     inp10 = LD_UB(src);
1304     src += src_stride;
1305     res0 = APPLY_VERT_QPEL_FILTER(inp6, inp5, inp4, inp3,
1306                                   inp7, inp8, inp9, inp10,
1307                                   const20, const6, const3);
1308     ST_UB(res0, dst);
1309     dst += dst_stride;
1310
1311     inp11 = LD_UB(src);
1312     src += src_stride;
1313     res0 = APPLY_VERT_QPEL_FILTER(inp7, inp6, inp5, inp4,
1314                                   inp8, inp9, inp10, inp11,
1315                                   const20, const6, const3);
1316     ST_UB(res0, dst);
1317     dst += dst_stride;
1318
1319     inp12 = LD_UB(src);
1320     src += src_stride;
1321     res0 = APPLY_VERT_QPEL_FILTER(inp8, inp7, inp6, inp5,
1322                                   inp9, inp10, inp11, inp12,
1323                                   const20, const6, const3);
1324     ST_UB(res0, dst);
1325     dst += dst_stride;
1326
1327     inp13 = LD_UB(src);
1328     src += src_stride;
1329     res0 = APPLY_VERT_QPEL_FILTER(inp9, inp8, inp7, inp6,
1330                                   inp10, inp11, inp12, inp13,
1331                                   const20, const6, const3);
1332     ST_UB(res0, dst);
1333     dst += dst_stride;
1334
1335     inp14 = LD_UB(src);
1336     src += src_stride;
1337     res0 = APPLY_VERT_QPEL_FILTER(inp10, inp9, inp8, inp7,
1338                                   inp11, inp12, inp13, inp14,
1339                                   const20, const6, const3);
1340     ST_UB(res0, dst);
1341     dst += dst_stride;
1342
1343     inp15 = LD_UB(src);
1344     src += src_stride;
1345     res0 = APPLY_VERT_QPEL_FILTER(inp11, inp10, inp9, inp8,
1346                                   inp12, inp13, inp14, inp15,
1347                                   const20, const6, const3);
1348     ST_UB(res0, dst);
1349     dst += dst_stride;
1350
1351     inp16 = LD_UB(src);
1352     res0 = APPLY_VERT_QPEL_FILTER(inp12, inp11, inp10, inp9,
1353                                   inp13, inp14, inp15, inp16,
1354                                   const20, const6, const3);
1355     ST_UB(res0, dst);
1356     dst += dst_stride;
1357
1358     res0 = APPLY_VERT_QPEL_FILTER(inp13, inp12, inp11, inp10,
1359                                   inp14, inp15, inp16, inp16,
1360                                   const20, const6, const3);
1361     ST_UB(res0, dst);
1362     dst += dst_stride;
1363
1364     res0 = APPLY_VERT_QPEL_FILTER(inp14, inp13, inp12, inp11,
1365                                   inp15, inp16, inp16, inp15,
1366                                   const20, const6, const3);
1367     ST_UB(res0, dst);
1368     dst += dst_stride;
1369
1370     res0 = APPLY_VERT_QPEL_FILTER(inp15, inp14, inp13, inp12,
1371                                   inp16, inp16, inp15, inp14,
1372                                   const20, const6, const3);
1373     ST_UB(res0, dst);
1374     dst += dst_stride;
1375 }
1376
1377 static void vert_mc_qpel_aver_src1_8x8_msa(const uint8_t *src,
1378                                            int32_t src_stride,
1379                                            uint8_t *dst,
1380                                            int32_t dst_stride)
1381 {
1382     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1383     v16u8 tmp0, tmp1, res0, res1;
1384     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1385     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1386     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1387
1388     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
1389     src += (4 * src_stride);
1390     LD_UB2(src, src_stride, inp4, inp5);
1391     src += (2 * src_stride);
1392     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp0, inp0, inp1, inp2,
1393                                         inp1, inp2, inp3, inp4,
1394                                         inp1, inp0, inp0, inp1,
1395                                         inp2, inp3, inp4, inp5,
1396                                         const20, const6, const3);
1397
1398     LD_UB2(src, src_stride, inp6, inp7);
1399     src += (2 * src_stride);
1400     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp2, inp1, inp0, inp0,
1401                                         inp3, inp4, inp5, inp6,
1402                                         inp3, inp2, inp1, inp0,
1403                                         inp4, inp5, inp6, inp7,
1404                                         const20, const6, const3);
1405     tmp0 = (v16u8) __msa_insve_d((v2i64) inp1, 1, (v2i64) inp2);
1406     tmp1 = (v16u8) __msa_insve_d((v2i64) inp3, 1, (v2i64) inp4);
1407     AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
1408     ST8x4_UB(res0, res1, dst, dst_stride);
1409     dst += (4 * dst_stride);
1410
1411     inp8 = LD_UB(src);
1412     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp4, inp3, inp2, inp1,
1413                                         inp5, inp6, inp7, inp8,
1414                                         inp5, inp4, inp3, inp2,
1415                                         inp6, inp7, inp8, inp8,
1416                                         const20, const6, const3);
1417     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp6, inp5, inp4, inp3,
1418                                         inp7, inp8, inp8, inp7,
1419                                         inp7, inp6, inp5, inp4,
1420                                         inp8, inp8, inp7, inp6,
1421                                         const20, const6, const3);
1422     tmp0 = (v16u8) __msa_insve_d((v2i64) inp5, 1, (v2i64) inp6);
1423     tmp1 = (v16u8) __msa_insve_d((v2i64) inp7, 1, (v2i64) inp8);
1424     AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
1425     ST8x4_UB(res0, res1, dst, dst_stride);
1426 }
1427
1428 static void vert_mc_qpel_aver_src1_16x16_msa(const uint8_t *src,
1429                                              int32_t src_stride,
1430                                              uint8_t *dst,
1431                                              int32_t dst_stride)
1432 {
1433     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1434     v16u8 inp9, inp10, inp11, inp12, inp13, inp14, inp15, inp16;
1435     v16u8 res0;
1436     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1437     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1438     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1439
1440     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
1441     src += (4 * src_stride);
1442     inp4 = LD_UB(src);
1443     src += src_stride;
1444     res0 = APPLY_VERT_QPEL_FILTER(inp0, inp0, inp1, inp2,
1445                                   inp1, inp2, inp3, inp4,
1446                                   const20, const6, const3);
1447     res0 = __msa_aver_u_b(res0, inp1);
1448     ST_UB(res0, dst);
1449     dst += dst_stride;
1450
1451     inp5 = LD_UB(src);
1452     src += src_stride;
1453     res0 = APPLY_VERT_QPEL_FILTER(inp1, inp0, inp0, inp1,
1454                                   inp2, inp3, inp4, inp5,
1455                                   const20, const6, const3);
1456     res0 = __msa_aver_u_b(res0, inp2);
1457     ST_UB(res0, dst);
1458     dst += dst_stride;
1459
1460     inp6 = LD_UB(src);
1461     src += src_stride;
1462     res0 = APPLY_VERT_QPEL_FILTER(inp2, inp1, inp0, inp0,
1463                                   inp3, inp4, inp5, inp6,
1464                                   const20, const6, const3);
1465     res0 = __msa_aver_u_b(res0, inp3);
1466     ST_UB(res0, dst);
1467     dst += dst_stride;
1468
1469     inp7 = LD_UB(src);
1470     src += src_stride;
1471     res0 = APPLY_VERT_QPEL_FILTER(inp3, inp2, inp1, inp0,
1472                                   inp4, inp5, inp6, inp7,
1473                                   const20, const6, const3);
1474     res0 = __msa_aver_u_b(res0, inp4);
1475     ST_UB(res0, dst);
1476     dst += dst_stride;
1477
1478     inp8 = LD_UB(src);
1479     src += src_stride;
1480     res0 = APPLY_VERT_QPEL_FILTER(inp4, inp3, inp2, inp1,
1481                                   inp5, inp6, inp7, inp8,
1482                                   const20, const6, const3);
1483     res0 = __msa_aver_u_b(res0, inp5);
1484     ST_UB(res0, dst);
1485     dst += dst_stride;
1486
1487     inp9 = LD_UB(src);
1488     src += src_stride;
1489     res0 = APPLY_VERT_QPEL_FILTER(inp5, inp4, inp3, inp2,
1490                                   inp6, inp7, inp8, inp9,
1491                                   const20, const6, const3);
1492     res0 = __msa_aver_u_b(res0, inp6);
1493     ST_UB(res0, dst);
1494     dst += dst_stride;
1495
1496     inp10 = LD_UB(src);
1497     src += src_stride;
1498     res0 = APPLY_VERT_QPEL_FILTER(inp6, inp5, inp4, inp3,
1499                                   inp7, inp8, inp9, inp10,
1500                                   const20, const6, const3);
1501     res0 = __msa_aver_u_b(res0, inp7);
1502     ST_UB(res0, dst);
1503     dst += dst_stride;
1504
1505     inp11 = LD_UB(src);
1506     src += src_stride;
1507     res0 = APPLY_VERT_QPEL_FILTER(inp7, inp6, inp5, inp4,
1508                                   inp8, inp9, inp10, inp11,
1509                                   const20, const6, const3);
1510     res0 = __msa_aver_u_b(res0, inp8);
1511     ST_UB(res0, dst);
1512     dst += dst_stride;
1513
1514     inp12 = LD_UB(src);
1515     src += src_stride;
1516     res0 = APPLY_VERT_QPEL_FILTER(inp8, inp7, inp6, inp5,
1517                                   inp9, inp10, inp11, inp12,
1518                                   const20, const6, const3);
1519     res0 = __msa_aver_u_b(res0, inp9);
1520     ST_UB(res0, dst);
1521     dst += dst_stride;
1522
1523     inp13 = LD_UB(src);
1524     src += src_stride;
1525     res0 = APPLY_VERT_QPEL_FILTER(inp9, inp8, inp7, inp6,
1526                                   inp10, inp11, inp12, inp13,
1527                                   const20, const6, const3);
1528     res0 = __msa_aver_u_b(res0, inp10);
1529     ST_UB(res0, dst);
1530     dst += dst_stride;
1531
1532     inp14 = LD_UB(src);
1533     src += src_stride;
1534     res0 = APPLY_VERT_QPEL_FILTER(inp10, inp9, inp8, inp7,
1535                                   inp11, inp12, inp13, inp14,
1536                                   const20, const6, const3);
1537     res0 = __msa_aver_u_b(res0, inp11);
1538     ST_UB(res0, dst);
1539     dst += dst_stride;
1540
1541     inp15 = LD_UB(src);
1542     src += src_stride;
1543     res0 = APPLY_VERT_QPEL_FILTER(inp11, inp10, inp9, inp8,
1544                                   inp12, inp13, inp14, inp15,
1545                                   const20, const6, const3);
1546     res0 = __msa_aver_u_b(res0, inp12);
1547     ST_UB(res0, dst);
1548     dst += dst_stride;
1549
1550     inp16 = LD_UB(src);
1551     res0 = APPLY_VERT_QPEL_FILTER(inp12, inp11, inp10, inp9,
1552                                   inp13, inp14, inp15, inp16,
1553                                   const20, const6, const3);
1554     res0 = __msa_aver_u_b(res0, inp13);
1555     ST_UB(res0, dst);
1556     dst += dst_stride;
1557
1558     res0 = APPLY_VERT_QPEL_FILTER(inp13, inp12, inp11, inp10,
1559                                   inp14, inp15, inp16, inp16,
1560                                   const20, const6, const3);
1561     res0 = __msa_aver_u_b(res0, inp14);
1562     ST_UB(res0, dst);
1563     dst += dst_stride;
1564
1565     res0 = APPLY_VERT_QPEL_FILTER(inp14, inp13, inp12, inp11,
1566                                   inp15, inp16, inp16, inp15,
1567                                   const20, const6, const3);
1568     res0 = __msa_aver_u_b(res0, inp15);
1569     ST_UB(res0, dst);
1570     dst += dst_stride;
1571
1572     res0 = APPLY_VERT_QPEL_FILTER(inp15, inp14, inp13, inp12,
1573                                   inp16, inp16, inp15, inp14,
1574                                   const20, const6, const3);
1575     res0 = __msa_aver_u_b(res0, inp16);
1576     ST_UB(res0, dst);
1577 }
1578
1579 static void vert_mc_qpel_no_rnd_aver_src0_8x8_msa(const uint8_t *src,
1580                                                   int32_t src_stride,
1581                                                   uint8_t *dst,
1582                                                   int32_t dst_stride)
1583 {
1584     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1585     v16u8 tmp0, tmp1, res0, res1;
1586     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1587     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1588     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1589
1590     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
1591     src += (4 * src_stride);
1592     LD_UB2(src, src_stride, inp4, inp5);
1593     src += (2 * src_stride);
1594     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp0, inp0, inp1, inp2,
1595                                                  inp1, inp2, inp3, inp4,
1596                                                  inp1, inp0, inp0, inp1,
1597                                                  inp2, inp3, inp4, inp5,
1598                                                  const20, const6, const3);
1599     LD_UB2(src, src_stride, inp6, inp7);
1600     src += (2 * src_stride);
1601     res1 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp2, inp1, inp0, inp0,
1602                                                  inp3, inp4, inp5, inp6,
1603                                                  inp3, inp2, inp1, inp0,
1604                                                  inp4, inp5, inp6, inp7,
1605                                                  const20, const6, const3);
1606     tmp0 = (v16u8) __msa_insve_d((v2i64) inp0, 1, (v2i64) inp1);
1607     tmp1 = (v16u8) __msa_insve_d((v2i64) inp2, 1, (v2i64) inp3);
1608     res0 = __msa_ave_u_b(res0, tmp0);
1609     res1 = __msa_ave_u_b(res1, tmp1);
1610     ST8x4_UB(res0, res1, dst, dst_stride);
1611     dst += (4 * dst_stride);
1612
1613     inp8 = LD_UB(src);
1614     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp4, inp3, inp2, inp1,
1615                                                  inp5, inp6, inp7, inp8,
1616                                                  inp5, inp4, inp3, inp2,
1617                                                  inp6, inp7, inp8, inp8,
1618                                                  const20, const6, const3);
1619     res1 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp6, inp5, inp4, inp3,
1620                                                  inp7, inp8, inp8, inp7,
1621                                                  inp7, inp6, inp5, inp4,
1622                                                  inp8, inp8, inp7, inp6,
1623                                                  const20, const6, const3);
1624     tmp0 = (v16u8) __msa_insve_d((v2i64) inp4, 1, (v2i64) inp5);
1625     tmp1 = (v16u8) __msa_insve_d((v2i64) inp6, 1, (v2i64) inp7);
1626     res0 = __msa_ave_u_b(res0, tmp0);
1627     res1 = __msa_ave_u_b(res1, tmp1);
1628     ST8x4_UB(res0, res1, dst, dst_stride);
1629     dst += (4 * dst_stride);
1630 }
1631
1632 static void vert_mc_qpel_no_rnd_aver_src0_16x16_msa(const uint8_t *src,
1633                                                     int32_t src_stride,
1634                                                     uint8_t *dst,
1635                                                     int32_t dst_stride)
1636 {
1637     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1638     v16u8 inp9, inp10, inp11, inp12, inp13, inp14, inp15, inp16;
1639     v16u8 res0;
1640     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1641     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1642     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1643
1644     LD_UB5(src, src_stride, inp0, inp1, inp2, inp3, inp4);
1645     src += (5 * src_stride);
1646     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp0, inp0, inp1, inp2,
1647                                            inp1, inp2, inp3, inp4,
1648                                            const20, const6, const3);
1649     res0 = __msa_ave_u_b(res0, inp0);
1650     ST_UB(res0, dst);
1651     dst += dst_stride;
1652
1653     inp5 = LD_UB(src);
1654     src += src_stride;
1655     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp1, inp0, inp0, inp1,
1656                                            inp2, inp3, inp4, inp5,
1657                                            const20, const6, const3);
1658     res0 = __msa_ave_u_b(res0, inp1);
1659     ST_UB(res0, dst);
1660     dst += dst_stride;
1661
1662     inp6 = LD_UB(src);
1663     src += src_stride;
1664     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp2, inp1, inp0, inp0,
1665                                            inp3, inp4, inp5, inp6,
1666                                            const20, const6, const3);
1667     res0 = __msa_ave_u_b(res0, inp2);
1668     ST_UB(res0, dst);
1669     dst += dst_stride;
1670
1671     inp7 = LD_UB(src);
1672     src += src_stride;
1673     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp3, inp2, inp1, inp0,
1674                                            inp4, inp5, inp6, inp7,
1675                                            const20, const6, const3);
1676     res0 = __msa_ave_u_b(res0, inp3);
1677     ST_UB(res0, dst);
1678     dst += dst_stride;
1679
1680     inp8 = LD_UB(src);
1681     src += src_stride;
1682     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp4, inp3, inp2, inp1,
1683                                            inp5, inp6, inp7, inp8,
1684                                            const20, const6, const3);
1685     res0 = __msa_ave_u_b(res0, inp4);
1686     ST_UB(res0, dst);
1687     dst += dst_stride;
1688
1689     inp9 = LD_UB(src);
1690     src += src_stride;
1691     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp5, inp4, inp3, inp2,
1692                                            inp6, inp7, inp8, inp9,
1693                                            const20, const6, const3);
1694     res0 = __msa_ave_u_b(res0, inp5);
1695     ST_UB(res0, dst);
1696     dst += dst_stride;
1697
1698     inp10 = LD_UB(src);
1699     src += src_stride;
1700     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp6, inp5, inp4, inp3,
1701                                            inp7, inp8, inp9, inp10,
1702                                            const20, const6, const3);
1703     res0 = __msa_ave_u_b(res0, inp6);
1704     ST_UB(res0, dst);
1705     dst += dst_stride;
1706
1707     inp11 = LD_UB(src);
1708     src += src_stride;
1709     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp7, inp6, inp5, inp4,
1710                                            inp8, inp9, inp10, inp11,
1711                                            const20, const6, const3);
1712     res0 = __msa_ave_u_b(res0, inp7);
1713     ST_UB(res0, dst);
1714     dst += dst_stride;
1715
1716     inp12 = LD_UB(src);
1717     src += src_stride;
1718     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp8, inp7, inp6, inp5,
1719                                            inp9, inp10, inp11, inp12,
1720                                            const20, const6, const3);
1721     res0 = __msa_ave_u_b(res0, inp8);
1722     ST_UB(res0, dst);
1723     dst += dst_stride;
1724
1725     inp13 = LD_UB(src);
1726     src += src_stride;
1727     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp9, inp8, inp7, inp6,
1728                                            inp10, inp11, inp12, inp13,
1729                                            const20, const6, const3);
1730     res0 = __msa_ave_u_b(res0, inp9);
1731     ST_UB(res0, dst);
1732     dst += dst_stride;
1733
1734     inp14 = LD_UB(src);
1735     src += src_stride;
1736     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp10, inp9, inp8, inp7,
1737                                            inp11, inp12, inp13, inp14,
1738                                            const20, const6, const3);
1739     res0 = __msa_ave_u_b(res0, inp10);
1740     ST_UB(res0, dst);
1741     dst += dst_stride;
1742
1743     inp15 = LD_UB(src);
1744     src += src_stride;
1745     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp11, inp10, inp9, inp8,
1746                                            inp12, inp13, inp14, inp15,
1747                                            const20, const6, const3);
1748     res0 = __msa_ave_u_b(res0, inp11);
1749     ST_UB(res0, dst);
1750     dst += dst_stride;
1751
1752     inp16 = LD_UB(src);
1753     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp12, inp11, inp10, inp9,
1754                                            inp13, inp14, inp15, inp16,
1755                                            const20, const6, const3);
1756     res0 = __msa_ave_u_b(res0, inp12);
1757     ST_UB(res0, dst);
1758     dst += dst_stride;
1759
1760     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp13, inp12, inp11, inp10,
1761                                            inp14, inp15, inp16, inp16,
1762                                            const20, const6, const3);
1763     res0 = __msa_ave_u_b(res0, inp13);
1764     ST_UB(res0, dst);
1765     dst += dst_stride;
1766
1767     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp14, inp13, inp12, inp11,
1768                                            inp15, inp16, inp16, inp15,
1769                                            const20, const6, const3);
1770     res0 = __msa_ave_u_b(res0, inp14);
1771     ST_UB(res0, dst);
1772     dst += dst_stride;
1773
1774     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp15, inp14, inp13, inp12,
1775                                            inp16, inp16, inp15, inp14,
1776                                            const20, const6, const3);
1777     res0 = __msa_ave_u_b(res0, inp15);
1778     ST_UB(res0, dst);
1779     dst += dst_stride;
1780 }
1781
1782 static void vert_mc_qpel_no_rnd_8x8_msa(const uint8_t *src,
1783                                         int32_t src_stride,
1784                                         uint8_t *dst,
1785                                         int32_t dst_stride)
1786 {
1787     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1788     v16u8 res0, res1;
1789     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1790     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1791     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1792
1793     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
1794     src += (4 * src_stride);
1795     LD_UB2(src, src_stride, inp4, inp5);
1796     src += (2 * src_stride);
1797     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp0, inp0, inp1, inp2,
1798                                                  inp1, inp2, inp3, inp4,
1799                                                  inp1, inp0, inp0, inp1,
1800                                                  inp2, inp3, inp4, inp5,
1801                                                  const20, const6, const3);
1802     LD_UB2(src, src_stride, inp6, inp7);
1803     src += (2 * src_stride);
1804     res1 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp2, inp1, inp0, inp0,
1805                                                  inp3, inp4, inp5, inp6,
1806                                                  inp3, inp2, inp1, inp0,
1807                                                  inp4, inp5, inp6, inp7,
1808                                                  const20, const6, const3);
1809     ST8x4_UB(res0, res1, dst, dst_stride);
1810     dst += (4 * dst_stride);
1811
1812     inp8 = LD_UB(src);
1813     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp4, inp3, inp2, inp1,
1814                                                  inp5, inp6, inp7, inp8,
1815                                                  inp5, inp4, inp3, inp2,
1816                                                  inp6, inp7, inp8, inp8,
1817                                                  const20, const6, const3);
1818     res1 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp6, inp5, inp4, inp3,
1819                                                  inp7, inp8, inp8, inp7,
1820                                                  inp7, inp6, inp5, inp4,
1821                                                  inp8, inp8, inp7, inp6,
1822                                                  const20, const6, const3);
1823     ST8x4_UB(res0, res1, dst, dst_stride);
1824     dst += (4 * dst_stride);
1825 }
1826
1827 static void vert_mc_qpel_no_rnd_16x16_msa(const uint8_t *src,
1828                                           int32_t src_stride,
1829                                           uint8_t *dst,
1830                                           int32_t dst_stride)
1831 {
1832     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1833     v16u8 inp9, inp10, inp11, inp12, inp13, inp14, inp15, inp16;
1834     v16u8 res0;
1835     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1836     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1837     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1838
1839     LD_UB5(src, src_stride, inp0, inp1, inp2, inp3, inp4);
1840     src += (5 * src_stride);
1841     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp0, inp0, inp1, inp2,
1842                                            inp1, inp2, inp3, inp4,
1843                                            const20, const6, const3);
1844     ST_UB(res0, dst);
1845     dst += dst_stride;
1846
1847     inp5 = LD_UB(src);
1848     src += src_stride;
1849     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp1, inp0, inp0, inp1,
1850                                            inp2, inp3, inp4, inp5,
1851                                            const20, const6, const3);
1852     ST_UB(res0, dst);
1853     dst += dst_stride;
1854
1855     inp6 = LD_UB(src);
1856     src += src_stride;
1857     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp2, inp1, inp0, inp0,
1858                                            inp3, inp4, inp5, inp6,
1859                                            const20, const6, const3);
1860     ST_UB(res0, dst);
1861     dst += dst_stride;
1862
1863     inp7 = LD_UB(src);
1864     src += src_stride;
1865     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp3, inp2, inp1, inp0,
1866                                            inp4, inp5, inp6, inp7,
1867                                            const20, const6, const3);
1868     ST_UB(res0, dst);
1869     dst += dst_stride;
1870
1871     inp8 = LD_UB(src);
1872     src += src_stride;
1873     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp4, inp3, inp2, inp1,
1874                                            inp5, inp6, inp7, inp8,
1875                                            const20, const6, const3);
1876     ST_UB(res0, dst);
1877     dst += dst_stride;
1878
1879     inp9 = LD_UB(src);
1880     src += src_stride;
1881     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp5, inp4, inp3, inp2,
1882                                            inp6, inp7, inp8, inp9,
1883                                            const20, const6, const3);
1884     ST_UB(res0, dst);
1885     dst += dst_stride;
1886
1887     inp10 = LD_UB(src);
1888     src += src_stride;
1889     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp6, inp5, inp4, inp3,
1890                                            inp7, inp8, inp9, inp10,
1891                                            const20, const6, const3);
1892     ST_UB(res0, dst);
1893     dst += dst_stride;
1894
1895     inp11 = LD_UB(src);
1896     src += src_stride;
1897     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp7, inp6, inp5, inp4,
1898                                            inp8, inp9, inp10, inp11,
1899                                            const20, const6, const3);
1900     ST_UB(res0, dst);
1901     dst += dst_stride;
1902
1903     inp12 = LD_UB(src);
1904     src += src_stride;
1905     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp8, inp7, inp6, inp5,
1906                                            inp9, inp10, inp11, inp12,
1907                                            const20, const6, const3);
1908     ST_UB(res0, dst);
1909     dst += dst_stride;
1910
1911     inp13 = LD_UB(src);
1912     src += src_stride;
1913     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp9, inp8, inp7, inp6,
1914                                            inp10, inp11, inp12, inp13,
1915                                            const20, const6, const3);
1916     ST_UB(res0, dst);
1917     dst += dst_stride;
1918
1919     inp14 = LD_UB(src);
1920     src += src_stride;
1921     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp10, inp9, inp8, inp7,
1922                                            inp11, inp12, inp13, inp14,
1923                                            const20, const6, const3);
1924     ST_UB(res0, dst);
1925     dst += dst_stride;
1926
1927     inp15 = LD_UB(src);
1928     src += src_stride;
1929     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp11, inp10, inp9, inp8,
1930                                            inp12, inp13, inp14, inp15,
1931                                            const20, const6, const3);
1932     ST_UB(res0, dst);
1933     dst += dst_stride;
1934
1935     inp16 = LD_UB(src);
1936     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp12, inp11, inp10, inp9,
1937                                            inp13, inp14, inp15, inp16,
1938                                            const20, const6, const3);
1939     ST_UB(res0, dst);
1940     dst += dst_stride;
1941
1942     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp13, inp12, inp11, inp10,
1943                                            inp14, inp15, inp16, inp16,
1944                                            const20, const6, const3);
1945     ST_UB(res0, dst);
1946     dst += dst_stride;
1947
1948     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp14, inp13, inp12, inp11,
1949                                            inp15, inp16, inp16, inp15,
1950                                            const20, const6, const3);
1951     ST_UB(res0, dst);
1952     dst += dst_stride;
1953
1954     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp15, inp14, inp13, inp12,
1955                                            inp16, inp16, inp15, inp14,
1956                                            const20, const6, const3);
1957     ST_UB(res0, dst);
1958 }
1959
1960 static void vert_mc_qpel_no_rnd_aver_src1_8x8_msa(const uint8_t *src,
1961                                                   int32_t src_stride,
1962                                                   uint8_t *dst,
1963                                                   int32_t dst_stride)
1964 {
1965     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
1966     v16u8 tmp0, tmp1, res0, res1;
1967     v16u8 const20 = (v16u8) __msa_ldi_b(20);
1968     v16u8 const6 = (v16u8) __msa_ldi_b(6);
1969     v16u8 const3 = (v16u8) __msa_ldi_b(3);
1970
1971     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
1972     src += (4 * src_stride);
1973     LD_UB2(src, src_stride, inp4, inp5);
1974     src += (2 * src_stride);
1975     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp0, inp0, inp1, inp2,
1976                                                  inp1, inp2, inp3, inp4,
1977                                                  inp1, inp0, inp0, inp1,
1978                                                  inp2, inp3, inp4, inp5,
1979                                                  const20, const6, const3);
1980     LD_UB2(src, src_stride, inp6, inp7);
1981     src += (2 * src_stride);
1982     res1 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp2, inp1, inp0, inp0,
1983                                                  inp3, inp4, inp5, inp6,
1984                                                  inp3, inp2, inp1, inp0,
1985                                                  inp4, inp5, inp6, inp7,
1986                                                  const20, const6, const3);
1987     tmp0 = (v16u8) __msa_insve_d((v2i64) inp1, 1, (v2i64) inp2);
1988     tmp1 = (v16u8) __msa_insve_d((v2i64) inp3, 1, (v2i64) inp4);
1989     res0 = __msa_ave_u_b(res0, tmp0);
1990     res1 = __msa_ave_u_b(res1, tmp1);
1991     ST8x4_UB(res0, res1, dst, dst_stride);
1992     dst += (4 * dst_stride);
1993
1994     inp8 = LD_UB(src);
1995     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp4, inp3, inp2, inp1,
1996                                                  inp5, inp6, inp7, inp8,
1997                                                  inp5, inp4, inp3, inp2,
1998                                                  inp6, inp7, inp8, inp8,
1999                                                  const20, const6, const3);
2000     res1 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(inp6, inp5, inp4, inp3,
2001                                                  inp7, inp8, inp8, inp7,
2002                                                  inp7, inp6, inp5, inp4,
2003                                                  inp8, inp8, inp7, inp6,
2004                                                  const20, const6, const3);
2005     tmp0 = (v16u8) __msa_insve_d((v2i64) inp5, 1, (v2i64) inp6);
2006     tmp1 = (v16u8) __msa_insve_d((v2i64) inp7, 1, (v2i64) inp8);
2007     res0 = __msa_ave_u_b(res0, tmp0);
2008     res1 = __msa_ave_u_b(res1, tmp1);
2009     ST8x4_UB(res0, res1, dst, dst_stride);
2010 }
2011
2012 static void vert_mc_qpel_no_rnd_aver_src1_16x16_msa(const uint8_t *src,
2013                                                     int32_t src_stride,
2014                                                     uint8_t *dst,
2015                                                     int32_t dst_stride)
2016 {
2017     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
2018     v16u8 inp9, inp10, inp11, inp12, inp13, inp14, inp15, inp16;
2019     v16u8 res0;
2020     v16u8 const20 = (v16u8) __msa_ldi_b(20);
2021     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2022     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2023
2024     LD_UB5(src, src_stride, inp0, inp1, inp2, inp3, inp4);
2025     src += (5 * src_stride);
2026     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp0, inp0, inp1, inp2,
2027                                            inp1, inp2, inp3, inp4,
2028                                            const20, const6, const3);
2029     res0 = __msa_ave_u_b(res0, inp1);
2030     ST_UB(res0, dst);
2031     dst += dst_stride;
2032
2033     inp5 = LD_UB(src);
2034     src += src_stride;
2035     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp1, inp0, inp0, inp1,
2036                                            inp2, inp3, inp4, inp5,
2037                                            const20, const6, const3);
2038     res0 = __msa_ave_u_b(res0, inp2);
2039     ST_UB(res0, dst);
2040     dst += dst_stride;
2041
2042     inp6 = LD_UB(src);
2043     src += src_stride;
2044     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp2, inp1, inp0, inp0,
2045                                            inp3, inp4, inp5, inp6,
2046                                            const20, const6, const3);
2047     res0 = __msa_ave_u_b(res0, inp3);
2048     ST_UB(res0, dst);
2049     dst += dst_stride;
2050
2051     inp7 = LD_UB(src);
2052     src += src_stride;
2053     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp3, inp2, inp1, inp0,
2054                                            inp4, inp5, inp6, inp7,
2055                                            const20, const6, const3);
2056     res0 = __msa_ave_u_b(res0, inp4);
2057     ST_UB(res0, dst);
2058     dst += dst_stride;
2059
2060     inp8 = LD_UB(src);
2061     src += src_stride;
2062     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp4, inp3, inp2, inp1,
2063                                            inp5, inp6, inp7, inp8,
2064                                            const20, const6, const3);
2065     res0 = __msa_ave_u_b(res0, inp5);
2066     ST_UB(res0, dst);
2067     dst += dst_stride;
2068
2069     inp9 = LD_UB(src);
2070     src += src_stride;
2071     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp5, inp4, inp3, inp2,
2072                                            inp6, inp7, inp8, inp9,
2073                                            const20, const6, const3);
2074     res0 = __msa_ave_u_b(res0, inp6);
2075     ST_UB(res0, dst);
2076     dst += dst_stride;
2077
2078     inp10 = LD_UB(src);
2079     src += src_stride;
2080     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp6, inp5, inp4, inp3,
2081                                            inp7, inp8, inp9, inp10,
2082                                            const20, const6, const3);
2083     res0 = __msa_ave_u_b(res0, inp7);
2084     ST_UB(res0, dst);
2085     dst += dst_stride;
2086
2087     inp11 = LD_UB(src);
2088     src += src_stride;
2089     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp7, inp6, inp5, inp4,
2090                                            inp8, inp9, inp10, inp11,
2091                                            const20, const6, const3);
2092     res0 = __msa_ave_u_b(res0, inp8);
2093     ST_UB(res0, dst);
2094     dst += dst_stride;
2095
2096     inp12 = LD_UB(src);
2097     src += src_stride;
2098     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp8, inp7, inp6, inp5,
2099                                            inp9, inp10, inp11, inp12,
2100                                            const20, const6, const3);
2101     res0 = __msa_ave_u_b(res0, inp9);
2102     ST_UB(res0, dst);
2103     dst += dst_stride;
2104
2105     inp13 = LD_UB(src);
2106     src += src_stride;
2107     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp9, inp8, inp7, inp6,
2108                                            inp10, inp11, inp12, inp13,
2109                                            const20, const6, const3);
2110     res0 = __msa_ave_u_b(res0, inp10);
2111     ST_UB(res0, dst);
2112     dst += dst_stride;
2113
2114     inp14 = LD_UB(src);
2115     src += src_stride;
2116     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp10, inp9, inp8, inp7,
2117                                            inp11, inp12, inp13, inp14,
2118                                            const20, const6, const3);
2119     res0 = __msa_ave_u_b(res0, inp11);
2120     ST_UB(res0, dst);
2121     dst += dst_stride;
2122
2123     inp15 = LD_UB(src);
2124     src += src_stride;
2125     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp11, inp10, inp9, inp8,
2126                                            inp12, inp13, inp14, inp15,
2127                                            const20, const6, const3);
2128     res0 = __msa_ave_u_b(res0, inp12);
2129     ST_UB(res0, dst);
2130     dst += dst_stride;
2131
2132     inp16 = LD_UB(src);
2133     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp12, inp11, inp10, inp9,
2134                                            inp13, inp14, inp15, inp16,
2135                                            const20, const6, const3);
2136     res0 = __msa_ave_u_b(res0, inp13);
2137     ST_UB(res0, dst);
2138     dst += dst_stride;
2139
2140     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp13, inp12, inp11, inp10,
2141                                            inp14, inp15, inp16, inp16,
2142                                            const20, const6, const3);
2143     res0 = __msa_ave_u_b(res0, inp14);
2144     ST_UB(res0, dst);
2145     dst += dst_stride;
2146
2147     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp14, inp13, inp12, inp11,
2148                                            inp15, inp16, inp16, inp15,
2149                                            const20, const6, const3);
2150     res0 = __msa_ave_u_b(res0, inp15);
2151     ST_UB(res0, dst);
2152     dst += dst_stride;
2153
2154     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER(inp15, inp14, inp13, inp12,
2155                                            inp16, inp16, inp15, inp14,
2156                                            const20, const6, const3);
2157     res0 = __msa_ave_u_b(res0, inp16);
2158     ST_UB(res0, dst);
2159 }
2160
2161 static void vert_mc_qpel_avg_dst_aver_src0_8x8_msa(const uint8_t *src,
2162                                                    int32_t src_stride,
2163                                                    uint8_t *dst,
2164                                                    int32_t dst_stride)
2165 {
2166     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
2167     v16u8 dst0, dst1, dst2, dst3;
2168     v16u8 tmp0, tmp1, res0, res1;
2169     v16u8 const20 = (v16u8) __msa_ldi_b(20);
2170     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2171     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2172
2173     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
2174     src += (4 * src_stride);
2175     LD_UB2(src, src_stride, inp4, inp5);
2176     src += (2 * src_stride);
2177     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp0, inp0, inp1, inp2,
2178                                         inp1, inp2, inp3, inp4,
2179                                         inp1, inp0, inp0, inp1,
2180                                         inp2, inp3, inp4, inp5,
2181                                         const20, const6, const3);
2182
2183     LD_UB2(src, src_stride, inp6, inp7);
2184     src += (2 * src_stride);
2185     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp2, inp1, inp0, inp0,
2186                                         inp3, inp4, inp5, inp6,
2187                                         inp3, inp2, inp1, inp0,
2188                                         inp4, inp5, inp6, inp7,
2189                                         const20, const6, const3);
2190
2191     LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
2192     tmp0 = (v16u8) __msa_insve_d((v2i64) inp0, 1, (v2i64) inp1);
2193     tmp1 = (v16u8) __msa_insve_d((v2i64) inp2, 1, (v2i64) inp3);
2194     dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
2195     dst2 = (v16u8) __msa_insve_d((v2i64) dst2, 1, (v2i64) dst3);
2196     AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
2197     AVER_UB2_UB(dst0, res0, dst2, res1, res0, res1);
2198     ST8x4_UB(res0, res1, dst, dst_stride);
2199     dst += (4 * dst_stride);
2200
2201     inp8 = LD_UB(src);
2202     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp4, inp3, inp2, inp1,
2203                                         inp5, inp6, inp7, inp8,
2204                                         inp5, inp4, inp3, inp2,
2205                                         inp6, inp7, inp8, inp8,
2206                                         const20, const6, const3);
2207     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp6, inp5, inp4, inp3,
2208                                         inp7, inp8, inp8, inp7,
2209                                         inp7, inp6, inp5, inp4,
2210                                         inp8, inp8, inp7, inp6,
2211                                         const20, const6, const3);
2212
2213     LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
2214     tmp0 = (v16u8) __msa_insve_d((v2i64) inp4, 1, (v2i64) inp5);
2215     tmp1 = (v16u8) __msa_insve_d((v2i64) inp6, 1, (v2i64) inp7);
2216     dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
2217     dst2 = (v16u8) __msa_insve_d((v2i64) dst2, 1, (v2i64) dst3);
2218     AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
2219     AVER_UB2_UB(dst0, res0, dst2, res1, res0, res1);
2220     ST8x4_UB(res0, res1, dst, dst_stride);
2221 }
2222
2223 static void vert_mc_qpel_avg_dst_aver_src0_16x16_msa(const uint8_t *src,
2224                                                      int32_t src_stride,
2225                                                      uint8_t *dst,
2226                                                      int32_t dst_stride)
2227 {
2228     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
2229     v16u8 inp9, inp10, inp11, inp12, inp13, inp14, inp15, inp16;
2230     v16u8 res0, res1, dst0, dst1;
2231     v16u8 const20 = (v16u8) __msa_ldi_b(20);
2232     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2233     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2234
2235     LD_UB5(src, src_stride, inp0, inp1, inp2, inp3, inp4);
2236     src += (5 * src_stride);
2237     res0 = APPLY_VERT_QPEL_FILTER(inp0, inp0, inp1, inp2,
2238                                   inp1, inp2, inp3, inp4,
2239                                   const20, const6, const3);
2240
2241     inp5 = LD_UB(src);
2242     src += src_stride;
2243     res1 = APPLY_VERT_QPEL_FILTER(inp1, inp0, inp0, inp1,
2244                                   inp2, inp3, inp4, inp5,
2245                                   const20, const6, const3);
2246
2247     LD_UB2(dst, dst_stride, dst0, dst1);
2248     AVER_UB2_UB(res0, inp0, res1, inp1, res0, res1);
2249     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2250     ST_UB2(res0, res1, dst, dst_stride);
2251     dst += (2 * dst_stride);
2252
2253     inp6 = LD_UB(src);
2254     src += src_stride;
2255     res0 = APPLY_VERT_QPEL_FILTER(inp2, inp1, inp0, inp0,
2256                                   inp3, inp4, inp5, inp6,
2257                                   const20, const6, const3);
2258
2259     inp7 = LD_UB(src);
2260     src += src_stride;
2261     res1 = APPLY_VERT_QPEL_FILTER(inp3, inp2, inp1, inp0,
2262                                   inp4, inp5, inp6, inp7,
2263                                   const20, const6, const3);
2264
2265     LD_UB2(dst, dst_stride, dst0, dst1);
2266     AVER_UB2_UB(res0, inp2, res1, inp3, res0, res1);
2267     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2268     ST_UB2(res0, res1, dst, dst_stride);
2269     dst += (2 * dst_stride);
2270
2271     LD_UB2(src, src_stride, inp8, inp9);
2272     src += (2 * src_stride);
2273     res0 = APPLY_VERT_QPEL_FILTER(inp4, inp3, inp2, inp1,
2274                                   inp5, inp6, inp7, inp8,
2275                                   const20, const6, const3);
2276     res1 = APPLY_VERT_QPEL_FILTER(inp5, inp4, inp3, inp2,
2277                                   inp6, inp7, inp8, inp9,
2278                                   const20, const6, const3);
2279
2280     LD_UB2(dst, dst_stride, dst0, dst1);
2281     AVER_UB2_UB(res0, inp4, res1, inp5, res0, res1);
2282     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2283     ST_UB2(res0, res1, dst, dst_stride);
2284     dst += (2 * dst_stride);
2285
2286     LD_UB2(src, src_stride, inp10, inp11);
2287     src += (2 * src_stride);
2288     res0 = APPLY_VERT_QPEL_FILTER(inp6, inp5, inp4, inp3,
2289                                   inp7, inp8, inp9, inp10,
2290                                   const20, const6, const3);
2291     res1 = APPLY_VERT_QPEL_FILTER(inp7, inp6, inp5, inp4,
2292                                   inp8, inp9, inp10, inp11,
2293                                   const20, const6, const3);
2294
2295     LD_UB2(dst, dst_stride, dst0, dst1);
2296     AVER_UB2_UB(res0, inp6, res1, inp7, res0, res1);
2297     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2298     ST_UB2(res0, res1, dst, dst_stride);
2299     dst += (2 * dst_stride);
2300
2301     LD_UB2(src, src_stride, inp12, inp13);
2302     src += (2 * src_stride);
2303     res0 = APPLY_VERT_QPEL_FILTER(inp8, inp7, inp6, inp5,
2304                                   inp9, inp10, inp11, inp12,
2305                                   const20, const6, const3);
2306     res1 = APPLY_VERT_QPEL_FILTER(inp9, inp8, inp7, inp6,
2307                                   inp10, inp11, inp12, inp13,
2308                                   const20, const6, const3);
2309     LD_UB2(dst, dst_stride, dst0, dst1);
2310     AVER_UB2_UB(res0, inp8, res1, inp9, res0, res1);
2311     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2312     ST_UB2(res0, res1, dst, dst_stride);
2313     dst += (2 * dst_stride);
2314
2315     LD_UB2(src, src_stride, inp14, inp15);
2316     src += (2 * src_stride);
2317     res0 = APPLY_VERT_QPEL_FILTER(inp10, inp9, inp8, inp7,
2318                                   inp11, inp12, inp13, inp14,
2319                                   const20, const6, const3);
2320     res1 = APPLY_VERT_QPEL_FILTER(inp11, inp10, inp9, inp8,
2321                                   inp12, inp13, inp14, inp15,
2322                                   const20, const6, const3);
2323
2324     LD_UB2(dst, dst_stride, dst0, dst1);
2325     AVER_UB2_UB(res0, inp10, res1, inp11, res0, res1);
2326     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2327     ST_UB2(res0, res1, dst, dst_stride);
2328     dst += (2 * dst_stride);
2329
2330     inp16 = LD_UB(src);
2331     res0 = APPLY_VERT_QPEL_FILTER(inp12, inp11, inp10, inp9,
2332                                   inp13, inp14, inp15, inp16,
2333                                   const20, const6, const3);
2334     res1 = APPLY_VERT_QPEL_FILTER(inp13, inp12, inp11, inp10,
2335                                   inp14, inp15, inp16, inp16,
2336                                   const20, const6, const3);
2337     LD_UB2(dst, dst_stride, dst0, dst1);
2338     AVER_UB2_UB(res0, inp12, res1, inp13, res0, res1);
2339     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2340     ST_UB2(res0, res1, dst, dst_stride);
2341     dst += (2 * dst_stride);
2342
2343     res0 = APPLY_VERT_QPEL_FILTER(inp14, inp13, inp12, inp11,
2344                                   inp15, inp16, inp16, inp15,
2345                                   const20, const6, const3);
2346     res1 = APPLY_VERT_QPEL_FILTER(inp15, inp14, inp13, inp12,
2347                                   inp16, inp16, inp15, inp14,
2348                                   const20, const6, const3);
2349     LD_UB2(dst, dst_stride, dst0, dst1);
2350     AVER_UB2_UB(res0, inp14, res1, inp15, res0, res1);
2351     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2352     ST_UB2(res0, res1, dst, dst_stride);
2353 }
2354
2355 static void vert_mc_qpel_avg_dst_8x8_msa(const uint8_t *src,
2356                                          int32_t src_stride,
2357                                          uint8_t *dst,
2358                                          int32_t dst_stride)
2359 {
2360     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
2361     v16u8 dst0, dst1, dst2, dst3;
2362     v16u8 res0, res1;
2363     v16u8 const20 = (v16u8) __msa_ldi_b(20);
2364     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2365     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2366
2367     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
2368     src += (4 * src_stride);
2369     LD_UB2(src, src_stride, inp4, inp5);
2370     src += (2 * src_stride);
2371     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp0, inp0, inp1, inp2,
2372                                         inp1, inp2, inp3, inp4,
2373                                         inp1, inp0, inp0, inp1,
2374                                         inp2, inp3, inp4, inp5,
2375                                         const20, const6, const3);
2376     LD_UB2(src, src_stride, inp6, inp7);
2377     src += (2 * src_stride);
2378     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp2, inp1, inp0, inp0,
2379                                         inp3, inp4, inp5, inp6,
2380                                         inp3, inp2, inp1, inp0,
2381                                         inp4, inp5, inp6, inp7,
2382                                         const20, const6, const3);
2383     LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
2384     dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
2385     dst2 = (v16u8) __msa_insve_d((v2i64) dst2, 1, (v2i64) dst3);
2386     AVER_UB2_UB(dst0, res0, dst2, res1, res0, res1);
2387     ST8x4_UB(res0, res1, dst, dst_stride);
2388     dst += (4 * dst_stride);
2389
2390     inp8 = LD_UB(src);
2391     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp4, inp3, inp2, inp1,
2392                                         inp5, inp6, inp7, inp8,
2393                                         inp5, inp4, inp3, inp2,
2394                                         inp6, inp7, inp8, inp8,
2395                                         const20, const6, const3);
2396     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp6, inp5, inp4, inp3,
2397                                         inp7, inp8, inp8, inp7,
2398                                         inp7, inp6, inp5, inp4,
2399                                         inp8, inp8, inp7, inp6,
2400                                         const20, const6, const3);
2401     LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
2402     dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
2403     dst2 = (v16u8) __msa_insve_d((v2i64) dst2, 1, (v2i64) dst3);
2404     AVER_UB2_UB(dst0, res0, dst2, res1, res0, res1);
2405     ST8x4_UB(res0, res1, dst, dst_stride);
2406     dst += (4 * dst_stride);
2407 }
2408
2409 static void vert_mc_qpel_avg_dst_16x16_msa(const uint8_t *src,
2410                                            int32_t src_stride,
2411                                            uint8_t *dst,
2412                                            int32_t dst_stride)
2413 {
2414     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
2415     v16u8 inp9, inp10, inp11, inp12, inp13, inp14, inp15, inp16;
2416     v16u8 res0, res1, dst0, dst1;
2417     v16u8 const20 = (v16u8) __msa_ldi_b(20);
2418     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2419     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2420
2421     LD_UB5(src, src_stride, inp0, inp1, inp2, inp3, inp4);
2422     src += (5 * src_stride);
2423     res0 = APPLY_VERT_QPEL_FILTER(inp0, inp0, inp1, inp2,
2424                                   inp1, inp2, inp3, inp4,
2425                                   const20, const6, const3);
2426     inp5 = LD_UB(src);
2427     src += src_stride;
2428     res1 = APPLY_VERT_QPEL_FILTER(inp1, inp0, inp0, inp1,
2429                                   inp2, inp3, inp4, inp5,
2430                                   const20, const6, const3);
2431     LD_UB2(dst, dst_stride, dst0, dst1);
2432     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2433     ST_UB2(res0, res1, dst, dst_stride);
2434     dst += (2 * dst_stride);
2435
2436     inp6 = LD_UB(src);
2437     src += src_stride;
2438     res0 = APPLY_VERT_QPEL_FILTER(inp2, inp1, inp0, inp0,
2439                                   inp3, inp4, inp5, inp6,
2440                                   const20, const6, const3);
2441     inp7 = LD_UB(src);
2442     src += src_stride;
2443     res1 = APPLY_VERT_QPEL_FILTER(inp3, inp2, inp1, inp0,
2444                                   inp4, inp5, inp6, inp7,
2445                                   const20, const6, const3);
2446     LD_UB2(dst, dst_stride, dst0, dst1);
2447     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2448     ST_UB2(res0, res1, dst, dst_stride);
2449     dst += (2 * dst_stride);
2450
2451     inp8 = LD_UB(src);
2452     src += src_stride;
2453     res0 = APPLY_VERT_QPEL_FILTER(inp4, inp3, inp2, inp1,
2454                                   inp5, inp6, inp7, inp8,
2455                                   const20, const6, const3);
2456     inp9 = LD_UB(src);
2457     src += src_stride;
2458     res1 = APPLY_VERT_QPEL_FILTER(inp5, inp4, inp3, inp2,
2459                                   inp6, inp7, inp8, inp9,
2460                                   const20, const6, const3);
2461     LD_UB2(dst, dst_stride, dst0, dst1);
2462     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2463     ST_UB2(res0, res1, dst, dst_stride);
2464     dst += (2 * dst_stride);
2465
2466     inp10 = LD_UB(src);
2467     src += src_stride;
2468     res0 = APPLY_VERT_QPEL_FILTER(inp6, inp5, inp4, inp3,
2469                                   inp7, inp8, inp9, inp10,
2470                                   const20, const6, const3);
2471     inp11 = LD_UB(src);
2472     src += src_stride;
2473     res1 = APPLY_VERT_QPEL_FILTER(inp7, inp6, inp5, inp4,
2474                                   inp8, inp9, inp10, inp11,
2475                                   const20, const6, const3);
2476     LD_UB2(dst, dst_stride, dst0, dst1);
2477     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2478     ST_UB2(res0, res1, dst, dst_stride);
2479     dst += (2 * dst_stride);
2480
2481     inp12 = LD_UB(src);
2482     src += src_stride;
2483     res0 = APPLY_VERT_QPEL_FILTER(inp8, inp7, inp6, inp5,
2484                                   inp9, inp10, inp11, inp12,
2485                                   const20, const6, const3);
2486     inp13 = LD_UB(src);
2487     src += src_stride;
2488     res1 = APPLY_VERT_QPEL_FILTER(inp9, inp8, inp7, inp6,
2489                                   inp10, inp11, inp12, inp13,
2490                                   const20, const6, const3);
2491     LD_UB2(dst, dst_stride, dst0, dst1);
2492     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2493     ST_UB2(res0, res1, dst, dst_stride);
2494     dst += (2 * dst_stride);
2495
2496     inp14 = LD_UB(src);
2497     src += src_stride;
2498     res0 = APPLY_VERT_QPEL_FILTER(inp10, inp9, inp8, inp7,
2499                                   inp11, inp12, inp13, inp14,
2500                                   const20, const6, const3);
2501     inp15 = LD_UB(src);
2502     src += src_stride;
2503     res1 = APPLY_VERT_QPEL_FILTER(inp11, inp10, inp9, inp8,
2504                                   inp12, inp13, inp14, inp15,
2505                                   const20, const6, const3);
2506     LD_UB2(dst, dst_stride, dst0, dst1);
2507     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2508     ST_UB2(res0, res1, dst, dst_stride);
2509     dst += (2 * dst_stride);
2510
2511     inp16 = LD_UB(src);
2512     res0 = APPLY_VERT_QPEL_FILTER(inp12, inp11, inp10, inp9,
2513                                   inp13, inp14, inp15, inp16,
2514                                   const20, const6, const3);
2515     res1 = APPLY_VERT_QPEL_FILTER(inp13, inp12, inp11, inp10,
2516                                   inp14, inp15, inp16, inp16,
2517                                   const20, const6, const3);
2518     LD_UB2(dst, dst_stride, dst0, dst1);
2519     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2520     ST_UB2(res0, res1, dst, dst_stride);
2521     dst += (2 * dst_stride);
2522
2523     res0 = APPLY_VERT_QPEL_FILTER(inp14, inp13, inp12, inp11,
2524                                   inp15, inp16, inp16, inp15,
2525                                   const20, const6, const3);
2526     res1 = APPLY_VERT_QPEL_FILTER(inp15, inp14, inp13, inp12,
2527                                   inp16, inp16, inp15, inp14,
2528                                   const20, const6, const3);
2529     LD_UB2(dst, dst_stride, dst0, dst1);
2530     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2531     ST_UB2(res0, res1, dst, dst_stride);
2532 }
2533
2534 static void vert_mc_qpel_avg_dst_aver_src1_8x8_msa(const uint8_t *src,
2535                                                    int32_t src_stride,
2536                                                    uint8_t *dst,
2537                                                    int32_t dst_stride)
2538 {
2539     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
2540     v16u8 dst0, dst1, dst2, dst3;
2541     v16u8 tmp0, tmp1, res0, res1;
2542     v16u8 const20 = (v16u8) __msa_ldi_b(20);
2543     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2544     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2545
2546     LD_UB4(src, src_stride, inp0, inp1, inp2, inp3);
2547     src += (4 * src_stride);
2548     LD_UB2(src, src_stride, inp4, inp5);
2549     src += (2 * src_stride);
2550     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp0, inp0, inp1, inp2,
2551                                         inp1, inp2, inp3, inp4,
2552                                         inp1, inp0, inp0, inp1,
2553                                         inp2, inp3, inp4, inp5,
2554                                         const20, const6, const3);
2555     LD_UB2(src, src_stride, inp6, inp7);
2556     src += (2 * src_stride);
2557     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp2, inp1, inp0, inp0,
2558                                         inp3, inp4, inp5, inp6,
2559                                         inp3, inp2, inp1, inp0,
2560                                         inp4, inp5, inp6, inp7,
2561                                         const20, const6, const3);
2562     LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
2563     tmp0 = (v16u8) __msa_insve_d((v2i64) inp1, 1, (v2i64) inp2);
2564     tmp1 = (v16u8) __msa_insve_d((v2i64) inp3, 1, (v2i64) inp4);
2565     dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
2566     dst2 = (v16u8) __msa_insve_d((v2i64) dst2, 1, (v2i64) dst3);
2567     AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
2568     AVER_UB2_UB(dst0, res0, dst2, res1, res0, res1);
2569     ST8x4_UB(res0, res1, dst, dst_stride);
2570     dst += (4 * dst_stride);
2571
2572     inp8 = LD_UB(src);
2573     res0 = APPLY_VERT_QPEL_FILTER_8BYTE(inp4, inp3, inp2, inp1,
2574                                         inp5, inp6, inp7, inp8,
2575                                         inp5, inp4, inp3, inp2,
2576                                         inp6, inp7, inp8, inp8,
2577                                         const20, const6, const3);
2578     res1 = APPLY_VERT_QPEL_FILTER_8BYTE(inp6, inp5, inp4, inp3,
2579                                         inp7, inp8, inp8, inp7,
2580                                         inp7, inp6, inp5, inp4,
2581                                         inp8, inp8, inp7, inp6,
2582                                         const20, const6, const3);
2583     LD_UB4(dst, dst_stride, dst0, dst1, dst2, dst3);
2584     tmp0 = (v16u8) __msa_insve_d((v2i64) inp5, 1, (v2i64) inp6);
2585     tmp1 = (v16u8) __msa_insve_d((v2i64) inp7, 1, (v2i64) inp8);
2586     dst0 = (v16u8) __msa_insve_d((v2i64) dst0, 1, (v2i64) dst1);
2587     dst2 = (v16u8) __msa_insve_d((v2i64) dst2, 1, (v2i64) dst3);
2588     AVER_UB2_UB(res0, tmp0, res1, tmp1, res0, res1);
2589     AVER_UB2_UB(dst0, res0, dst2, res1, res0, res1);
2590     ST8x4_UB(res0, res1, dst, dst_stride);
2591 }
2592
2593 static void vert_mc_qpel_avg_dst_aver_src1_16x16_msa(const uint8_t *src,
2594                                                      int32_t src_stride,
2595                                                      uint8_t *dst,
2596                                                      int32_t dst_stride)
2597 {
2598     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7, inp8;
2599     v16u8 inp9, inp10, inp11, inp12, inp13, inp14, inp15, inp16;
2600     v16u8 res0, res1, dst0, dst1;
2601     v16u8 const20 = (v16u8) __msa_ldi_b(20);
2602     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2603     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2604
2605     LD_UB5(src, src_stride, inp0, inp1, inp2, inp3, inp4);
2606     src += (5 * src_stride);
2607     res0 = APPLY_VERT_QPEL_FILTER(inp0, inp0, inp1, inp2,
2608                                   inp1, inp2, inp3, inp4,
2609                                   const20, const6, const3);
2610     inp5 = LD_UB(src);
2611     src += src_stride;
2612     res1 = APPLY_VERT_QPEL_FILTER(inp1, inp0, inp0, inp1,
2613                                   inp2, inp3, inp4, inp5,
2614                                   const20, const6, const3);
2615     LD_UB2(dst, dst_stride, dst0, dst1);
2616     AVER_UB2_UB(res0, inp1, res1, inp2, res0, res1);
2617     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2618     ST_UB2(res0, res1, dst, dst_stride);
2619     dst += (2 * dst_stride);
2620
2621     inp6 = LD_UB(src);
2622     src += src_stride;
2623     res0 = APPLY_VERT_QPEL_FILTER(inp2, inp1, inp0, inp0,
2624                                   inp3, inp4, inp5, inp6,
2625                                   const20, const6, const3);
2626     inp7 = LD_UB(src);
2627     src += src_stride;
2628     res1 = APPLY_VERT_QPEL_FILTER(inp3, inp2, inp1, inp0,
2629                                   inp4, inp5, inp6, inp7,
2630                                   const20, const6, const3);
2631     LD_UB2(dst, dst_stride, dst0, dst1);
2632     AVER_UB2_UB(res0, inp3, res1, inp4, res0, res1);
2633     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2634     ST_UB2(res0, res1, dst, dst_stride);
2635     dst += (2 * dst_stride);
2636
2637     inp8 = LD_UB(src);
2638     src += src_stride;
2639     res0 = APPLY_VERT_QPEL_FILTER(inp4, inp3, inp2, inp1,
2640                                   inp5, inp6, inp7, inp8,
2641                                   const20, const6, const3);
2642     inp9 = LD_UB(src);
2643     src += src_stride;
2644     res1 = APPLY_VERT_QPEL_FILTER(inp5, inp4, inp3, inp2,
2645                                   inp6, inp7, inp8, inp9,
2646                                   const20, const6, const3);
2647     LD_UB2(dst, dst_stride, dst0, dst1);
2648     AVER_UB2_UB(res0, inp5, res1, inp6, res0, res1);
2649     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2650     ST_UB2(res0, res1, dst, dst_stride);
2651     dst += (2 * dst_stride);
2652
2653     inp10 = LD_UB(src);
2654     src += src_stride;
2655     res0 = APPLY_VERT_QPEL_FILTER(inp6, inp5, inp4, inp3,
2656                                   inp7, inp8, inp9, inp10,
2657                                   const20, const6, const3);
2658     inp11 = LD_UB(src);
2659     src += src_stride;
2660     res1 = APPLY_VERT_QPEL_FILTER(inp7, inp6, inp5, inp4,
2661                                   inp8, inp9, inp10, inp11,
2662                                   const20, const6, const3);
2663     LD_UB2(dst, dst_stride, dst0, dst1);
2664     AVER_UB2_UB(res0, inp7, res1, inp8, res0, res1);
2665     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2666     ST_UB2(res0, res1, dst, dst_stride);
2667     dst += (2 * dst_stride);
2668
2669     inp12 = LD_UB(src);
2670     src += src_stride;
2671     res0 = APPLY_VERT_QPEL_FILTER(inp8, inp7, inp6, inp5,
2672                                   inp9, inp10, inp11, inp12,
2673                                   const20, const6, const3);
2674     inp13 = LD_UB(src);
2675     src += src_stride;
2676     res1 = APPLY_VERT_QPEL_FILTER(inp9, inp8, inp7, inp6,
2677                                   inp10, inp11, inp12, inp13,
2678                                   const20, const6, const3);
2679     LD_UB2(dst, dst_stride, dst0, dst1);
2680     AVER_UB2_UB(res0, inp9, res1, inp10, res0, res1);
2681     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2682     ST_UB2(res0, res1, dst, dst_stride);
2683     dst += (2 * dst_stride);
2684
2685     inp14 = LD_UB(src);
2686     src += src_stride;
2687     res0 = APPLY_VERT_QPEL_FILTER(inp10, inp9, inp8, inp7,
2688                                   inp11, inp12, inp13, inp14,
2689                                   const20, const6, const3);
2690     inp15 = LD_UB(src);
2691     src += src_stride;
2692     res1 = APPLY_VERT_QPEL_FILTER(inp11, inp10, inp9, inp8,
2693                                   inp12, inp13, inp14, inp15,
2694                                   const20, const6, const3);
2695     LD_UB2(dst, dst_stride, dst0, dst1);
2696     AVER_UB2_UB(res0, inp11, res1, inp12, res0, res1);
2697     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2698     ST_UB2(res0, res1, dst, dst_stride);
2699     dst += (2 * dst_stride);
2700
2701     inp16 = LD_UB(src);
2702     res0 = APPLY_VERT_QPEL_FILTER(inp12, inp11, inp10, inp9,
2703                                   inp13, inp14, inp15, inp16,
2704                                   const20, const6, const3);
2705     res1 = APPLY_VERT_QPEL_FILTER(inp13, inp12, inp11, inp10,
2706                                   inp14, inp15, inp16, inp16,
2707                                   const20, const6, const3);
2708     LD_UB2(dst, dst_stride, dst0, dst1);
2709     AVER_UB2_UB(res0, inp13, res1, inp14, res0, res1);
2710     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2711     ST_UB2(res0, res1, dst, dst_stride);
2712     dst += (2 * dst_stride);
2713
2714     res0 = APPLY_VERT_QPEL_FILTER(inp14, inp13, inp12, inp11,
2715                                   inp15, inp16, inp16, inp15,
2716                                   const20, const6, const3);
2717     res1 = APPLY_VERT_QPEL_FILTER(inp15, inp14, inp13, inp12,
2718                                   inp16, inp16, inp15, inp14,
2719                                   const20, const6, const3);
2720     LD_UB2(dst, dst_stride, dst0, dst1);
2721     AVER_UB2_UB(res0, inp15, res1, inp16, res0, res1);
2722     AVER_UB2_UB(res0, dst0, res1, dst1, res0, res1);
2723     ST_UB2(res0, res1, dst, dst_stride);
2724 }
2725
2726 static void hv_mc_qpel_no_rnd_horiz_src0_16x16_msa(const uint8_t *src,
2727                                                    int32_t src_stride,
2728                                                    uint8_t *dst,
2729                                                    int32_t dst_stride,
2730                                                    int32_t height)
2731 {
2732     uint8_t loop_count;
2733     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
2734     v16u8 res;
2735     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
2736     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2737     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2738     v8u16 const20 = (v8u16) __msa_ldi_h(20);
2739
2740     for (loop_count = (height >> 2); loop_count--;) {
2741         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
2742         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
2743         src += (4 * src_stride);
2744         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp0, inp1, mask,
2745                                                const20, const6, const3);
2746         res = __msa_ave_u_b(inp0, res);
2747         ST_UB(res, dst);
2748         dst += dst_stride;
2749
2750         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp2, inp3, mask,
2751                                                const20, const6, const3);
2752         res = __msa_ave_u_b(inp2, res);
2753         ST_UB(res, dst);
2754         dst += dst_stride;
2755
2756         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp4, inp5, mask,
2757                                                const20, const6, const3);
2758         res = __msa_ave_u_b(inp4, res);
2759         ST_UB(res, dst);
2760         dst += dst_stride;
2761
2762         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp6, inp7, mask,
2763                                                const20, const6, const3);
2764         res = __msa_ave_u_b(inp6, res);
2765         ST_UB(res, dst);
2766         dst += dst_stride;
2767     }
2768
2769     LD_UB2(src, 1, inp0, inp1);
2770     res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp0, inp1, mask,
2771                                            const20, const6, const3);
2772     res = __msa_ave_u_b(inp0, res);
2773     ST_UB(res, dst);
2774 }
2775
2776 static void hv_mc_qpel_no_rnd_aver_hv_src00_16x16_msa(const uint8_t *src,
2777                                                       int32_t src_stride,
2778                                                       uint8_t *dst,
2779                                                       int32_t dst_stride)
2780 {
2781     uint8_t buff[272];
2782
2783     hv_mc_qpel_no_rnd_horiz_src0_16x16_msa(src, src_stride, buff, 16, 16);
2784     vert_mc_qpel_no_rnd_aver_src0_16x16_msa(buff, 16, dst, dst_stride);
2785 }
2786
2787 static void hv_mc_qpel_no_rnd_aver_hv_src00_8x8_msa(const uint8_t *src,
2788                                                     int32_t src_stride,
2789                                                     uint8_t *dst,
2790                                                     int32_t dst_stride)
2791 {
2792     v16u8 inp0, inp1, inp2, inp3;
2793     v16u8 res0, res1, avg0, avg1;
2794     v16u8 horiz0, horiz1, horiz2, horiz3;
2795     v16u8 horiz4, horiz5, horiz6, horiz7, horiz8;
2796     v16u8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
2797     v16u8 mask1 = { 0, 2, 0, 3, 1, 4, 2, 5, 3, 6, 4, 7, 5, 8, 6, 8 };
2798     v16u8 mask2 = { 1, 3, 0, 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 8, 5, 7 };
2799     v16u8 mask3 = { 2, 4, 1, 5, 0, 6, 0, 7, 1, 8, 2, 8, 3, 7, 4, 6 };
2800     v16u8 const20 = (v16u8) __msa_ldi_b(20);
2801     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2802     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2803
2804     LD_UB2(src, src_stride, inp0, inp1);
2805     src += (2 * src_stride);
2806     res0 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp0, inp1, mask0, mask1,
2807                                                   mask2, mask3, const20,
2808                                                   const6, const3);
2809     inp0 = (v16u8) __msa_ilvr_d((v2i64) inp1, (v2i64) inp0);
2810     horiz0 = __msa_ave_u_b(inp0, res0);
2811     horiz1 = (v16u8) __msa_splati_d((v2i64) horiz0, 1);
2812     LD_UB2(src, src_stride, inp2, inp3);
2813     src += (2 * src_stride);
2814     res1 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp2, inp3, mask0, mask1,
2815                                                   mask2, mask3, const20,
2816                                                   const6, const3);
2817     inp2 = (v16u8) __msa_ilvr_d((v2i64) inp3, (v2i64) inp2);
2818     horiz2 = __msa_ave_u_b(inp2, res1);
2819     horiz3 = (v16u8) __msa_splati_d((v2i64) horiz2, 1);
2820     LD_UB2(src, src_stride, inp0, inp1);
2821     src += (2 * src_stride);
2822     res0 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp0, inp1, mask0, mask1,
2823                                                   mask2, mask3, const20,
2824                                                   const6, const3);
2825     inp0 = (v16u8) __msa_ilvr_d((v2i64) inp1, (v2i64) inp0);
2826     horiz4 = __msa_ave_u_b(inp0, res0);
2827     horiz5 = (v16u8) __msa_splati_d((v2i64) horiz4, 1);
2828     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(horiz0, horiz0, horiz1, horiz2,
2829                                                  horiz1, horiz2, horiz3, horiz4,
2830                                                  horiz1, horiz0, horiz0, horiz1,
2831                                                  horiz2, horiz3, horiz4, horiz5,
2832                                                  const20, const6, const3);
2833     avg0 = (v16u8) __msa_ilvr_d((v2i64) horiz1, (v2i64) horiz0);
2834     res0 = __msa_ave_u_b(avg0, res0);
2835     ST8x2_UB(res0, dst, dst_stride);
2836     dst += (2 * dst_stride);
2837
2838     LD_UB2(src, src_stride, inp2, inp3);
2839     src += (2 * src_stride);
2840     res1 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE(inp2, inp3, mask0, mask1,
2841                                                   mask2, mask3, const20,
2842                                                   const6, const3);
2843     inp2 = (v16u8) __msa_ilvr_d((v2i64) inp3, (v2i64) inp2);
2844     horiz6 = __msa_ave_u_b(inp2, res1);
2845     horiz7 = (v16u8) __msa_splati_d((v2i64) horiz6, 1);
2846     inp0 = LD_UB(src);
2847     res0 = APPLY_HORIZ_QPEL_NO_ROUND_FILTER_8BYTE_1ROW(inp0, mask0, mask1,
2848                                                        mask2, mask3, const20,
2849                                                        const6, const3);
2850     horiz8 = __msa_ave_u_b(inp0, res0);
2851     res1 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(horiz2, horiz1, horiz0, horiz0,
2852                                                  horiz3, horiz4, horiz5, horiz6,
2853                                                  horiz3, horiz2, horiz1, horiz0,
2854                                                  horiz4, horiz5, horiz6, horiz7,
2855                                                  const20, const6, const3);
2856     avg1 = (v16u8) __msa_ilvr_d((v2i64) horiz3, (v2i64) horiz2);
2857     res1 = __msa_ave_u_b(avg1, res1);
2858     res0 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(horiz4, horiz3, horiz2, horiz1,
2859                                                  horiz5, horiz6, horiz7, horiz8,
2860                                                  horiz5, horiz4, horiz3, horiz2,
2861                                                  horiz6, horiz7, horiz8, horiz8,
2862                                                  const20, const6, const3);
2863     ST8x2_UB(res1, dst, dst_stride);
2864     dst += 2 * dst_stride;
2865
2866     avg0 = (v16u8) __msa_ilvr_d((v2i64) horiz5, (v2i64) horiz4);
2867     res0 = __msa_ave_u_b(avg0, res0);
2868     res1 = APPLY_VERT_QPEL_NO_ROUND_FILTER_8BYTE(horiz6, horiz5, horiz4, horiz3,
2869                                                  horiz7, horiz8, horiz8, horiz7,
2870                                                  horiz7, horiz6, horiz5, horiz4,
2871                                                  horiz8, horiz8, horiz7, horiz6,
2872                                                  const20, const6, const3);
2873     ST8x2_UB(res0, dst, dst_stride);
2874     dst += 2 * dst_stride;
2875
2876     avg1 = (v16u8) __msa_ilvr_d((v2i64) horiz7, (v2i64) horiz6);
2877     res1 = __msa_ave_u_b(avg1, res1);
2878     ST8x2_UB(res1, dst, dst_stride);
2879 }
2880
2881 static void hv_mc_qpel_no_rnd_horiz_16x16_msa(const uint8_t *src,
2882                                               int32_t src_stride,
2883                                               uint8_t *dst,
2884                                               int32_t dst_stride,
2885                                               int32_t height)
2886 {
2887     uint8_t loop_count;
2888     v16u8 inp0, inp1, inp2, inp3, inp4, inp5, inp6, inp7;
2889     v16u8 res;
2890     v16u8 mask = { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 };
2891     v16u8 const6 = (v16u8) __msa_ldi_b(6);
2892     v16u8 const3 = (v16u8) __msa_ldi_b(3);
2893     v8u16 const20 = (v8u16) __msa_ldi_h(20);
2894
2895     for (loop_count = (height >> 2); loop_count--;) {
2896         LD_UB4(src, src_stride, inp0, inp2, inp4, inp6);
2897         LD_UB4((src + 1), src_stride, inp1, inp3, inp5, inp7);
2898         src += (4 * src_stride);
2899         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp0, inp1, mask,
2900                                                const20, const6, const3);
2901         ST_UB(res, dst);
2902         dst += dst_stride;
2903
2904         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp2, inp3, mask,
2905                                                const20, const6, const3);
2906         ST_UB(res, dst);
2907         dst += dst_stride;
2908
2909         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp4, inp5, mask,
2910                                                const20, const6, const3);
2911         ST_UB(res, dst);
2912         dst += dst_stride;
2913
2914         res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp6, inp7, mask,
2915                                                const20, const6, const3);
2916         ST_UB(res, dst);
2917         dst += dst_stride;
2918     }
2919
2920     LD_UB2(src, 1, inp0, inp1);
2921     res = APPLY_HORIZ_QPEL_NO_ROUND_FILTER(inp0, inp1, mask,
2922                                            const20, const6, const3);
2923     ST_UB(res, dst);
2924 }
2925
2926 static void hv_mc_qpel_no_rnd_aver_v_src0_16x16_msa(const uint8_t *src,
2927                                                     int32_t src_stride,
2928                                                     uint8_t *dst,
2929                                                     int32_t dst_stride)
2930 {
2931     uint8_t buff[272];
2932
2933     hv_mc_qpel_no_rnd_horiz_16x16_msa(src, src_stride, buff, 16, 16);
2934     vert_mc_qpel_no_rnd_aver_src0_16x16_msa(buff, 16, dst, dst_stride);
2935 }
2936
2937 static void hv_mc_qpel_no_rnd_aver_v_src0_8x8_msa(const uint8_t *src,
2938                                                   int32_t src_stride,
2939                                                   uint8_t *dst,
2940                                                   int32_t dst_stride)
2941 {