2 * Copyright (C) 2001-2003 Michael Niedermayer <michaelni@gmx.at>
4 * This file is part of Libav.
6 * Libav is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * Libav is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with Libav; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "swscale_template.h"
28 #if COMPILE_TEMPLATE_AMD3DNOW
29 #define PREFETCH "prefetch"
30 #elif COMPILE_TEMPLATE_MMX2
31 #define PREFETCH "prefetchnta"
33 #define PREFETCH " # nop"
36 #if COMPILE_TEMPLATE_MMX2
37 #define PAVGB(a,b) "pavgb " #a ", " #b " \n\t"
38 #elif COMPILE_TEMPLATE_AMD3DNOW
39 #define PAVGB(a,b) "pavgusb " #a ", " #b " \n\t"
42 #if COMPILE_TEMPLATE_MMX2
43 #define REAL_MOVNTQ(a,b) "movntq " #a ", " #b " \n\t"
45 #define REAL_MOVNTQ(a,b) "movq " #a ", " #b " \n\t"
47 #define MOVNTQ(a,b) REAL_MOVNTQ(a,b)
49 #define YSCALEYUV2YV12X(x, offset, dest, width) \
51 "xor %%"REG_a", %%"REG_a" \n\t"\
52 "movq "VROUNDER_OFFSET"(%0), %%mm3 \n\t"\
53 "movq %%mm3, %%mm4 \n\t"\
54 "lea " offset "(%0), %%"REG_d" \n\t"\
55 "mov (%%"REG_d"), %%"REG_S" \n\t"\
56 ".p2align 4 \n\t" /* FIXME Unroll? */\
58 "movq 8(%%"REG_d"), %%mm0 \n\t" /* filterCoeff */\
59 "movq " x "(%%"REG_S", %%"REG_a", 2), %%mm2 \n\t" /* srcData */\
60 "movq 8+" x "(%%"REG_S", %%"REG_a", 2), %%mm5 \n\t" /* srcData */\
61 "add $16, %%"REG_d" \n\t"\
62 "mov (%%"REG_d"), %%"REG_S" \n\t"\
63 "test %%"REG_S", %%"REG_S" \n\t"\
64 "pmulhw %%mm0, %%mm2 \n\t"\
65 "pmulhw %%mm0, %%mm5 \n\t"\
66 "paddw %%mm2, %%mm3 \n\t"\
67 "paddw %%mm5, %%mm4 \n\t"\
69 "psraw $3, %%mm3 \n\t"\
70 "psraw $3, %%mm4 \n\t"\
71 "packuswb %%mm4, %%mm3 \n\t"\
72 MOVNTQ(%%mm3, (%1, %%REGa))\
73 "add $8, %%"REG_a" \n\t"\
74 "cmp %2, %%"REG_a" \n\t"\
75 "movq "VROUNDER_OFFSET"(%0), %%mm3 \n\t"\
76 "movq %%mm3, %%mm4 \n\t"\
77 "lea " offset "(%0), %%"REG_d" \n\t"\
78 "mov (%%"REG_d"), %%"REG_S" \n\t"\
80 :: "r" (&c->redDither),\
81 "r" (dest), "g" ((x86_reg)width)\
82 : "%"REG_a, "%"REG_d, "%"REG_S\
85 #define YSCALEYUV2YV12X_ACCURATE(x, offset, dest, width) \
87 "lea " offset "(%0), %%"REG_d" \n\t"\
88 "xor %%"REG_a", %%"REG_a" \n\t"\
89 "pxor %%mm4, %%mm4 \n\t"\
90 "pxor %%mm5, %%mm5 \n\t"\
91 "pxor %%mm6, %%mm6 \n\t"\
92 "pxor %%mm7, %%mm7 \n\t"\
93 "mov (%%"REG_d"), %%"REG_S" \n\t"\
96 "movq " x "(%%"REG_S", %%"REG_a", 2), %%mm0 \n\t" /* srcData */\
97 "movq 8+" x "(%%"REG_S", %%"REG_a", 2), %%mm2 \n\t" /* srcData */\
98 "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S" \n\t"\
99 "movq " x "(%%"REG_S", %%"REG_a", 2), %%mm1 \n\t" /* srcData */\
100 "movq %%mm0, %%mm3 \n\t"\
101 "punpcklwd %%mm1, %%mm0 \n\t"\
102 "punpckhwd %%mm1, %%mm3 \n\t"\
103 "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm1 \n\t" /* filterCoeff */\
104 "pmaddwd %%mm1, %%mm0 \n\t"\
105 "pmaddwd %%mm1, %%mm3 \n\t"\
106 "paddd %%mm0, %%mm4 \n\t"\
107 "paddd %%mm3, %%mm5 \n\t"\
108 "movq 8+" x "(%%"REG_S", %%"REG_a", 2), %%mm3 \n\t" /* srcData */\
109 "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S" \n\t"\
110 "add $"STR(APCK_SIZE)", %%"REG_d" \n\t"\
111 "test %%"REG_S", %%"REG_S" \n\t"\
112 "movq %%mm2, %%mm0 \n\t"\
113 "punpcklwd %%mm3, %%mm2 \n\t"\
114 "punpckhwd %%mm3, %%mm0 \n\t"\
115 "pmaddwd %%mm1, %%mm2 \n\t"\
116 "pmaddwd %%mm1, %%mm0 \n\t"\
117 "paddd %%mm2, %%mm6 \n\t"\
118 "paddd %%mm0, %%mm7 \n\t"\
120 "psrad $16, %%mm4 \n\t"\
121 "psrad $16, %%mm5 \n\t"\
122 "psrad $16, %%mm6 \n\t"\
123 "psrad $16, %%mm7 \n\t"\
124 "movq "VROUNDER_OFFSET"(%0), %%mm0 \n\t"\
125 "packssdw %%mm5, %%mm4 \n\t"\
126 "packssdw %%mm7, %%mm6 \n\t"\
127 "paddw %%mm0, %%mm4 \n\t"\
128 "paddw %%mm0, %%mm6 \n\t"\
129 "psraw $3, %%mm4 \n\t"\
130 "psraw $3, %%mm6 \n\t"\
131 "packuswb %%mm6, %%mm4 \n\t"\
132 MOVNTQ(%%mm4, (%1, %%REGa))\
133 "add $8, %%"REG_a" \n\t"\
134 "cmp %2, %%"REG_a" \n\t"\
135 "lea " offset "(%0), %%"REG_d" \n\t"\
136 "pxor %%mm4, %%mm4 \n\t"\
137 "pxor %%mm5, %%mm5 \n\t"\
138 "pxor %%mm6, %%mm6 \n\t"\
139 "pxor %%mm7, %%mm7 \n\t"\
140 "mov (%%"REG_d"), %%"REG_S" \n\t"\
142 :: "r" (&c->redDither),\
143 "r" (dest), "g" ((x86_reg)width)\
144 : "%"REG_a, "%"REG_d, "%"REG_S\
147 #define YSCALEYUV2YV121 \
148 "mov %2, %%"REG_a" \n\t"\
149 ".p2align 4 \n\t" /* FIXME Unroll? */\
151 "movq (%0, %%"REG_a", 2), %%mm0 \n\t"\
152 "movq 8(%0, %%"REG_a", 2), %%mm1 \n\t"\
153 "psraw $7, %%mm0 \n\t"\
154 "psraw $7, %%mm1 \n\t"\
155 "packuswb %%mm1, %%mm0 \n\t"\
156 MOVNTQ(%%mm0, (%1, %%REGa))\
157 "add $8, %%"REG_a" \n\t"\
160 #define YSCALEYUV2YV121_ACCURATE \
161 "mov %2, %%"REG_a" \n\t"\
162 "pcmpeqw %%mm7, %%mm7 \n\t"\
163 "psrlw $15, %%mm7 \n\t"\
164 "psllw $6, %%mm7 \n\t"\
165 ".p2align 4 \n\t" /* FIXME Unroll? */\
167 "movq (%0, %%"REG_a", 2), %%mm0 \n\t"\
168 "movq 8(%0, %%"REG_a", 2), %%mm1 \n\t"\
169 "paddsw %%mm7, %%mm0 \n\t"\
170 "paddsw %%mm7, %%mm1 \n\t"\
171 "psraw $7, %%mm0 \n\t"\
172 "psraw $7, %%mm1 \n\t"\
173 "packuswb %%mm1, %%mm0 \n\t"\
174 MOVNTQ(%%mm0, (%1, %%REGa))\
175 "add $8, %%"REG_a" \n\t"\
179 :: "m" (-lumFilterSize), "m" (-chrFilterSize),
180 "m" (lumMmxFilter+lumFilterSize*4), "m" (chrMmxFilter+chrFilterSize*4),
181 "r" (dest), "m" (dstW_reg),
182 "m" (lumSrc+lumFilterSize), "m" (chrSrc+chrFilterSize)
183 : "%eax", "%ebx", "%ecx", "%edx", "%esi"
185 #define YSCALEYUV2PACKEDX_UV \
187 "xor %%"REG_a", %%"REG_a" \n\t"\
191 "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d" \n\t"\
192 "mov (%%"REG_d"), %%"REG_S" \n\t"\
193 "movq "VROUNDER_OFFSET"(%0), %%mm3 \n\t"\
194 "movq %%mm3, %%mm4 \n\t"\
197 "movq 8(%%"REG_d"), %%mm0 \n\t" /* filterCoeff */\
198 "movq (%%"REG_S", %%"REG_a"), %%mm2 \n\t" /* UsrcData */\
199 "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm5 \n\t" /* VsrcData */\
200 "add $16, %%"REG_d" \n\t"\
201 "mov (%%"REG_d"), %%"REG_S" \n\t"\
202 "pmulhw %%mm0, %%mm2 \n\t"\
203 "pmulhw %%mm0, %%mm5 \n\t"\
204 "paddw %%mm2, %%mm3 \n\t"\
205 "paddw %%mm5, %%mm4 \n\t"\
206 "test %%"REG_S", %%"REG_S" \n\t"\
209 #define YSCALEYUV2PACKEDX_YA(offset,coeff,src1,src2,dst1,dst2) \
210 "lea "offset"(%0), %%"REG_d" \n\t"\
211 "mov (%%"REG_d"), %%"REG_S" \n\t"\
212 "movq "VROUNDER_OFFSET"(%0), "#dst1" \n\t"\
213 "movq "#dst1", "#dst2" \n\t"\
216 "movq 8(%%"REG_d"), "#coeff" \n\t" /* filterCoeff */\
217 "movq (%%"REG_S", %%"REG_a", 2), "#src1" \n\t" /* Y1srcData */\
218 "movq 8(%%"REG_S", %%"REG_a", 2), "#src2" \n\t" /* Y2srcData */\
219 "add $16, %%"REG_d" \n\t"\
220 "mov (%%"REG_d"), %%"REG_S" \n\t"\
221 "pmulhw "#coeff", "#src1" \n\t"\
222 "pmulhw "#coeff", "#src2" \n\t"\
223 "paddw "#src1", "#dst1" \n\t"\
224 "paddw "#src2", "#dst2" \n\t"\
225 "test %%"REG_S", %%"REG_S" \n\t"\
228 #define YSCALEYUV2PACKEDX \
229 YSCALEYUV2PACKEDX_UV \
230 YSCALEYUV2PACKEDX_YA(LUM_MMX_FILTER_OFFSET,%%mm0,%%mm2,%%mm5,%%mm1,%%mm7) \
232 #define YSCALEYUV2PACKEDX_END \
233 :: "r" (&c->redDither), \
234 "m" (dummy), "m" (dummy), "m" (dummy),\
235 "r" (dest), "m" (dstW_reg) \
236 : "%"REG_a, "%"REG_d, "%"REG_S \
239 #define YSCALEYUV2PACKEDX_ACCURATE_UV \
241 "xor %%"REG_a", %%"REG_a" \n\t"\
245 "lea "CHR_MMX_FILTER_OFFSET"(%0), %%"REG_d" \n\t"\
246 "mov (%%"REG_d"), %%"REG_S" \n\t"\
247 "pxor %%mm4, %%mm4 \n\t"\
248 "pxor %%mm5, %%mm5 \n\t"\
249 "pxor %%mm6, %%mm6 \n\t"\
250 "pxor %%mm7, %%mm7 \n\t"\
253 "movq (%%"REG_S", %%"REG_a"), %%mm0 \n\t" /* UsrcData */\
254 "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm2 \n\t" /* VsrcData */\
255 "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S" \n\t"\
256 "movq (%%"REG_S", %%"REG_a"), %%mm1 \n\t" /* UsrcData */\
257 "movq %%mm0, %%mm3 \n\t"\
258 "punpcklwd %%mm1, %%mm0 \n\t"\
259 "punpckhwd %%mm1, %%mm3 \n\t"\
260 "movq "STR(APCK_COEF)"(%%"REG_d"),%%mm1 \n\t" /* filterCoeff */\
261 "pmaddwd %%mm1, %%mm0 \n\t"\
262 "pmaddwd %%mm1, %%mm3 \n\t"\
263 "paddd %%mm0, %%mm4 \n\t"\
264 "paddd %%mm3, %%mm5 \n\t"\
265 "movq "AV_STRINGIFY(VOF)"(%%"REG_S", %%"REG_a"), %%mm3 \n\t" /* VsrcData */\
266 "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S" \n\t"\
267 "add $"STR(APCK_SIZE)", %%"REG_d" \n\t"\
268 "test %%"REG_S", %%"REG_S" \n\t"\
269 "movq %%mm2, %%mm0 \n\t"\
270 "punpcklwd %%mm3, %%mm2 \n\t"\
271 "punpckhwd %%mm3, %%mm0 \n\t"\
272 "pmaddwd %%mm1, %%mm2 \n\t"\
273 "pmaddwd %%mm1, %%mm0 \n\t"\
274 "paddd %%mm2, %%mm6 \n\t"\
275 "paddd %%mm0, %%mm7 \n\t"\
277 "psrad $16, %%mm4 \n\t"\
278 "psrad $16, %%mm5 \n\t"\
279 "psrad $16, %%mm6 \n\t"\
280 "psrad $16, %%mm7 \n\t"\
281 "movq "VROUNDER_OFFSET"(%0), %%mm0 \n\t"\
282 "packssdw %%mm5, %%mm4 \n\t"\
283 "packssdw %%mm7, %%mm6 \n\t"\
284 "paddw %%mm0, %%mm4 \n\t"\
285 "paddw %%mm0, %%mm6 \n\t"\
286 "movq %%mm4, "U_TEMP"(%0) \n\t"\
287 "movq %%mm6, "V_TEMP"(%0) \n\t"\
289 #define YSCALEYUV2PACKEDX_ACCURATE_YA(offset) \
290 "lea "offset"(%0), %%"REG_d" \n\t"\
291 "mov (%%"REG_d"), %%"REG_S" \n\t"\
292 "pxor %%mm1, %%mm1 \n\t"\
293 "pxor %%mm5, %%mm5 \n\t"\
294 "pxor %%mm7, %%mm7 \n\t"\
295 "pxor %%mm6, %%mm6 \n\t"\
298 "movq (%%"REG_S", %%"REG_a", 2), %%mm0 \n\t" /* Y1srcData */\
299 "movq 8(%%"REG_S", %%"REG_a", 2), %%mm2 \n\t" /* Y2srcData */\
300 "mov "STR(APCK_PTR2)"(%%"REG_d"), %%"REG_S" \n\t"\
301 "movq (%%"REG_S", %%"REG_a", 2), %%mm4 \n\t" /* Y1srcData */\
302 "movq %%mm0, %%mm3 \n\t"\
303 "punpcklwd %%mm4, %%mm0 \n\t"\
304 "punpckhwd %%mm4, %%mm3 \n\t"\
305 "movq "STR(APCK_COEF)"(%%"REG_d"), %%mm4 \n\t" /* filterCoeff */\
306 "pmaddwd %%mm4, %%mm0 \n\t"\
307 "pmaddwd %%mm4, %%mm3 \n\t"\
308 "paddd %%mm0, %%mm1 \n\t"\
309 "paddd %%mm3, %%mm5 \n\t"\
310 "movq 8(%%"REG_S", %%"REG_a", 2), %%mm3 \n\t" /* Y2srcData */\
311 "mov "STR(APCK_SIZE)"(%%"REG_d"), %%"REG_S" \n\t"\
312 "add $"STR(APCK_SIZE)", %%"REG_d" \n\t"\
313 "test %%"REG_S", %%"REG_S" \n\t"\
314 "movq %%mm2, %%mm0 \n\t"\
315 "punpcklwd %%mm3, %%mm2 \n\t"\
316 "punpckhwd %%mm3, %%mm0 \n\t"\
317 "pmaddwd %%mm4, %%mm2 \n\t"\
318 "pmaddwd %%mm4, %%mm0 \n\t"\
319 "paddd %%mm2, %%mm7 \n\t"\
320 "paddd %%mm0, %%mm6 \n\t"\
322 "psrad $16, %%mm1 \n\t"\
323 "psrad $16, %%mm5 \n\t"\
324 "psrad $16, %%mm7 \n\t"\
325 "psrad $16, %%mm6 \n\t"\
326 "movq "VROUNDER_OFFSET"(%0), %%mm0 \n\t"\
327 "packssdw %%mm5, %%mm1 \n\t"\
328 "packssdw %%mm6, %%mm7 \n\t"\
329 "paddw %%mm0, %%mm1 \n\t"\
330 "paddw %%mm0, %%mm7 \n\t"\
331 "movq "U_TEMP"(%0), %%mm3 \n\t"\
332 "movq "V_TEMP"(%0), %%mm4 \n\t"\
334 #define YSCALEYUV2PACKEDX_ACCURATE \
335 YSCALEYUV2PACKEDX_ACCURATE_UV \
336 YSCALEYUV2PACKEDX_ACCURATE_YA(LUM_MMX_FILTER_OFFSET)
338 #define YSCALEYUV2RGBX \
339 "psubw "U_OFFSET"(%0), %%mm3 \n\t" /* (U-128)8*/\
340 "psubw "V_OFFSET"(%0), %%mm4 \n\t" /* (V-128)8*/\
341 "movq %%mm3, %%mm2 \n\t" /* (U-128)8*/\
342 "movq %%mm4, %%mm5 \n\t" /* (V-128)8*/\
343 "pmulhw "UG_COEFF"(%0), %%mm3 \n\t"\
344 "pmulhw "VG_COEFF"(%0), %%mm4 \n\t"\
345 /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
346 "pmulhw "UB_COEFF"(%0), %%mm2 \n\t"\
347 "pmulhw "VR_COEFF"(%0), %%mm5 \n\t"\
348 "psubw "Y_OFFSET"(%0), %%mm1 \n\t" /* 8(Y-16)*/\
349 "psubw "Y_OFFSET"(%0), %%mm7 \n\t" /* 8(Y-16)*/\
350 "pmulhw "Y_COEFF"(%0), %%mm1 \n\t"\
351 "pmulhw "Y_COEFF"(%0), %%mm7 \n\t"\
352 /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
353 "paddw %%mm3, %%mm4 \n\t"\
354 "movq %%mm2, %%mm0 \n\t"\
355 "movq %%mm5, %%mm6 \n\t"\
356 "movq %%mm4, %%mm3 \n\t"\
357 "punpcklwd %%mm2, %%mm2 \n\t"\
358 "punpcklwd %%mm5, %%mm5 \n\t"\
359 "punpcklwd %%mm4, %%mm4 \n\t"\
360 "paddw %%mm1, %%mm2 \n\t"\
361 "paddw %%mm1, %%mm5 \n\t"\
362 "paddw %%mm1, %%mm4 \n\t"\
363 "punpckhwd %%mm0, %%mm0 \n\t"\
364 "punpckhwd %%mm6, %%mm6 \n\t"\
365 "punpckhwd %%mm3, %%mm3 \n\t"\
366 "paddw %%mm7, %%mm0 \n\t"\
367 "paddw %%mm7, %%mm6 \n\t"\
368 "paddw %%mm7, %%mm3 \n\t"\
369 /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
370 "packuswb %%mm0, %%mm2 \n\t"\
371 "packuswb %%mm6, %%mm5 \n\t"\
372 "packuswb %%mm3, %%mm4 \n\t"\
374 #define REAL_YSCALEYUV2PACKED(index, c) \
375 "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0 \n\t"\
376 "movq "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm1 \n\t"\
377 "psraw $3, %%mm0 \n\t"\
378 "psraw $3, %%mm1 \n\t"\
379 "movq %%mm0, "CHR_MMX_FILTER_OFFSET"+8("#c") \n\t"\
380 "movq %%mm1, "LUM_MMX_FILTER_OFFSET"+8("#c") \n\t"\
381 "xor "#index", "#index" \n\t"\
384 "movq (%2, "#index"), %%mm2 \n\t" /* uvbuf0[eax]*/\
385 "movq (%3, "#index"), %%mm3 \n\t" /* uvbuf1[eax]*/\
386 "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5 \n\t" /* uvbuf0[eax+2048]*/\
387 "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4 \n\t" /* uvbuf1[eax+2048]*/\
388 "psubw %%mm3, %%mm2 \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
389 "psubw %%mm4, %%mm5 \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
390 "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0 \n\t"\
391 "pmulhw %%mm0, %%mm2 \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
392 "pmulhw %%mm0, %%mm5 \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
393 "psraw $7, %%mm3 \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
394 "psraw $7, %%mm4 \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
395 "paddw %%mm2, %%mm3 \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
396 "paddw %%mm5, %%mm4 \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
397 "movq (%0, "#index", 2), %%mm0 \n\t" /*buf0[eax]*/\
398 "movq (%1, "#index", 2), %%mm1 \n\t" /*buf1[eax]*/\
399 "movq 8(%0, "#index", 2), %%mm6 \n\t" /*buf0[eax]*/\
400 "movq 8(%1, "#index", 2), %%mm7 \n\t" /*buf1[eax]*/\
401 "psubw %%mm1, %%mm0 \n\t" /* buf0[eax] - buf1[eax]*/\
402 "psubw %%mm7, %%mm6 \n\t" /* buf0[eax] - buf1[eax]*/\
403 "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0 \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
404 "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6 \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
405 "psraw $7, %%mm1 \n\t" /* buf0[eax] - buf1[eax] >>4*/\
406 "psraw $7, %%mm7 \n\t" /* buf0[eax] - buf1[eax] >>4*/\
407 "paddw %%mm0, %%mm1 \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
408 "paddw %%mm6, %%mm7 \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
410 #define YSCALEYUV2PACKED(index, c) REAL_YSCALEYUV2PACKED(index, c)
412 #define REAL_YSCALEYUV2RGB_UV(index, c) \
413 "xor "#index", "#index" \n\t"\
416 "movq (%2, "#index"), %%mm2 \n\t" /* uvbuf0[eax]*/\
417 "movq (%3, "#index"), %%mm3 \n\t" /* uvbuf1[eax]*/\
418 "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5 \n\t" /* uvbuf0[eax+2048]*/\
419 "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4 \n\t" /* uvbuf1[eax+2048]*/\
420 "psubw %%mm3, %%mm2 \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\
421 "psubw %%mm4, %%mm5 \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\
422 "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0 \n\t"\
423 "pmulhw %%mm0, %%mm2 \n\t" /* (uvbuf0[eax] - uvbuf1[eax])uvalpha1>>16*/\
424 "pmulhw %%mm0, %%mm5 \n\t" /* (uvbuf0[eax+2048] - uvbuf1[eax+2048])uvalpha1>>16*/\
425 "psraw $4, %%mm3 \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
426 "psraw $4, %%mm4 \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
427 "paddw %%mm2, %%mm3 \n\t" /* uvbuf0[eax]uvalpha1 - uvbuf1[eax](1-uvalpha1)*/\
428 "paddw %%mm5, %%mm4 \n\t" /* uvbuf0[eax+2048]uvalpha1 - uvbuf1[eax+2048](1-uvalpha1)*/\
429 "psubw "U_OFFSET"("#c"), %%mm3 \n\t" /* (U-128)8*/\
430 "psubw "V_OFFSET"("#c"), %%mm4 \n\t" /* (V-128)8*/\
431 "movq %%mm3, %%mm2 \n\t" /* (U-128)8*/\
432 "movq %%mm4, %%mm5 \n\t" /* (V-128)8*/\
433 "pmulhw "UG_COEFF"("#c"), %%mm3 \n\t"\
434 "pmulhw "VG_COEFF"("#c"), %%mm4 \n\t"\
435 /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
437 #define REAL_YSCALEYUV2RGB_YA(index, c, b1, b2) \
438 "movq ("#b1", "#index", 2), %%mm0 \n\t" /*buf0[eax]*/\
439 "movq ("#b2", "#index", 2), %%mm1 \n\t" /*buf1[eax]*/\
440 "movq 8("#b1", "#index", 2), %%mm6 \n\t" /*buf0[eax]*/\
441 "movq 8("#b2", "#index", 2), %%mm7 \n\t" /*buf1[eax]*/\
442 "psubw %%mm1, %%mm0 \n\t" /* buf0[eax] - buf1[eax]*/\
443 "psubw %%mm7, %%mm6 \n\t" /* buf0[eax] - buf1[eax]*/\
444 "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm0 \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
445 "pmulhw "LUM_MMX_FILTER_OFFSET"+8("#c"), %%mm6 \n\t" /* (buf0[eax] - buf1[eax])yalpha1>>16*/\
446 "psraw $4, %%mm1 \n\t" /* buf0[eax] - buf1[eax] >>4*/\
447 "psraw $4, %%mm7 \n\t" /* buf0[eax] - buf1[eax] >>4*/\
448 "paddw %%mm0, %%mm1 \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
449 "paddw %%mm6, %%mm7 \n\t" /* buf0[eax]yalpha1 + buf1[eax](1-yalpha1) >>16*/\
451 #define REAL_YSCALEYUV2RGB_COEFF(c) \
452 "pmulhw "UB_COEFF"("#c"), %%mm2 \n\t"\
453 "pmulhw "VR_COEFF"("#c"), %%mm5 \n\t"\
454 "psubw "Y_OFFSET"("#c"), %%mm1 \n\t" /* 8(Y-16)*/\
455 "psubw "Y_OFFSET"("#c"), %%mm7 \n\t" /* 8(Y-16)*/\
456 "pmulhw "Y_COEFF"("#c"), %%mm1 \n\t"\
457 "pmulhw "Y_COEFF"("#c"), %%mm7 \n\t"\
458 /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
459 "paddw %%mm3, %%mm4 \n\t"\
460 "movq %%mm2, %%mm0 \n\t"\
461 "movq %%mm5, %%mm6 \n\t"\
462 "movq %%mm4, %%mm3 \n\t"\
463 "punpcklwd %%mm2, %%mm2 \n\t"\
464 "punpcklwd %%mm5, %%mm5 \n\t"\
465 "punpcklwd %%mm4, %%mm4 \n\t"\
466 "paddw %%mm1, %%mm2 \n\t"\
467 "paddw %%mm1, %%mm5 \n\t"\
468 "paddw %%mm1, %%mm4 \n\t"\
469 "punpckhwd %%mm0, %%mm0 \n\t"\
470 "punpckhwd %%mm6, %%mm6 \n\t"\
471 "punpckhwd %%mm3, %%mm3 \n\t"\
472 "paddw %%mm7, %%mm0 \n\t"\
473 "paddw %%mm7, %%mm6 \n\t"\
474 "paddw %%mm7, %%mm3 \n\t"\
475 /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
476 "packuswb %%mm0, %%mm2 \n\t"\
477 "packuswb %%mm6, %%mm5 \n\t"\
478 "packuswb %%mm3, %%mm4 \n\t"\
480 #define YSCALEYUV2RGB_YA(index, c, b1, b2) REAL_YSCALEYUV2RGB_YA(index, c, b1, b2)
482 #define YSCALEYUV2RGB(index, c) \
483 REAL_YSCALEYUV2RGB_UV(index, c) \
484 REAL_YSCALEYUV2RGB_YA(index, c, %0, %1) \
485 REAL_YSCALEYUV2RGB_COEFF(c)
487 #define REAL_YSCALEYUV2PACKED1(index, c) \
488 "xor "#index", "#index" \n\t"\
491 "movq (%2, "#index"), %%mm3 \n\t" /* uvbuf0[eax]*/\
492 "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4 \n\t" /* uvbuf0[eax+2048]*/\
493 "psraw $7, %%mm3 \n\t" \
494 "psraw $7, %%mm4 \n\t" \
495 "movq (%0, "#index", 2), %%mm1 \n\t" /*buf0[eax]*/\
496 "movq 8(%0, "#index", 2), %%mm7 \n\t" /*buf0[eax]*/\
497 "psraw $7, %%mm1 \n\t" \
498 "psraw $7, %%mm7 \n\t" \
500 #define YSCALEYUV2PACKED1(index, c) REAL_YSCALEYUV2PACKED1(index, c)
502 #define REAL_YSCALEYUV2RGB1(index, c) \
503 "xor "#index", "#index" \n\t"\
506 "movq (%2, "#index"), %%mm3 \n\t" /* uvbuf0[eax]*/\
507 "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm4 \n\t" /* uvbuf0[eax+2048]*/\
508 "psraw $4, %%mm3 \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\
509 "psraw $4, %%mm4 \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\
510 "psubw "U_OFFSET"("#c"), %%mm3 \n\t" /* (U-128)8*/\
511 "psubw "V_OFFSET"("#c"), %%mm4 \n\t" /* (V-128)8*/\
512 "movq %%mm3, %%mm2 \n\t" /* (U-128)8*/\
513 "movq %%mm4, %%mm5 \n\t" /* (V-128)8*/\
514 "pmulhw "UG_COEFF"("#c"), %%mm3 \n\t"\
515 "pmulhw "VG_COEFF"("#c"), %%mm4 \n\t"\
516 /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
517 "movq (%0, "#index", 2), %%mm1 \n\t" /*buf0[eax]*/\
518 "movq 8(%0, "#index", 2), %%mm7 \n\t" /*buf0[eax]*/\
519 "psraw $4, %%mm1 \n\t" /* buf0[eax] - buf1[eax] >>4*/\
520 "psraw $4, %%mm7 \n\t" /* buf0[eax] - buf1[eax] >>4*/\
521 "pmulhw "UB_COEFF"("#c"), %%mm2 \n\t"\
522 "pmulhw "VR_COEFF"("#c"), %%mm5 \n\t"\
523 "psubw "Y_OFFSET"("#c"), %%mm1 \n\t" /* 8(Y-16)*/\
524 "psubw "Y_OFFSET"("#c"), %%mm7 \n\t" /* 8(Y-16)*/\
525 "pmulhw "Y_COEFF"("#c"), %%mm1 \n\t"\
526 "pmulhw "Y_COEFF"("#c"), %%mm7 \n\t"\
527 /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
528 "paddw %%mm3, %%mm4 \n\t"\
529 "movq %%mm2, %%mm0 \n\t"\
530 "movq %%mm5, %%mm6 \n\t"\
531 "movq %%mm4, %%mm3 \n\t"\
532 "punpcklwd %%mm2, %%mm2 \n\t"\
533 "punpcklwd %%mm5, %%mm5 \n\t"\
534 "punpcklwd %%mm4, %%mm4 \n\t"\
535 "paddw %%mm1, %%mm2 \n\t"\
536 "paddw %%mm1, %%mm5 \n\t"\
537 "paddw %%mm1, %%mm4 \n\t"\
538 "punpckhwd %%mm0, %%mm0 \n\t"\
539 "punpckhwd %%mm6, %%mm6 \n\t"\
540 "punpckhwd %%mm3, %%mm3 \n\t"\
541 "paddw %%mm7, %%mm0 \n\t"\
542 "paddw %%mm7, %%mm6 \n\t"\
543 "paddw %%mm7, %%mm3 \n\t"\
544 /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
545 "packuswb %%mm0, %%mm2 \n\t"\
546 "packuswb %%mm6, %%mm5 \n\t"\
547 "packuswb %%mm3, %%mm4 \n\t"\
549 #define YSCALEYUV2RGB1(index, c) REAL_YSCALEYUV2RGB1(index, c)
551 #define REAL_YSCALEYUV2PACKED1b(index, c) \
552 "xor "#index", "#index" \n\t"\
555 "movq (%2, "#index"), %%mm2 \n\t" /* uvbuf0[eax]*/\
556 "movq (%3, "#index"), %%mm3 \n\t" /* uvbuf1[eax]*/\
557 "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5 \n\t" /* uvbuf0[eax+2048]*/\
558 "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4 \n\t" /* uvbuf1[eax+2048]*/\
559 "paddw %%mm2, %%mm3 \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
560 "paddw %%mm5, %%mm4 \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
561 "psrlw $8, %%mm3 \n\t" \
562 "psrlw $8, %%mm4 \n\t" \
563 "movq (%0, "#index", 2), %%mm1 \n\t" /*buf0[eax]*/\
564 "movq 8(%0, "#index", 2), %%mm7 \n\t" /*buf0[eax]*/\
565 "psraw $7, %%mm1 \n\t" \
566 "psraw $7, %%mm7 \n\t"
567 #define YSCALEYUV2PACKED1b(index, c) REAL_YSCALEYUV2PACKED1b(index, c)
569 // do vertical chrominance interpolation
570 #define REAL_YSCALEYUV2RGB1b(index, c) \
571 "xor "#index", "#index" \n\t"\
574 "movq (%2, "#index"), %%mm2 \n\t" /* uvbuf0[eax]*/\
575 "movq (%3, "#index"), %%mm3 \n\t" /* uvbuf1[eax]*/\
576 "movq "AV_STRINGIFY(VOF)"(%2, "#index"), %%mm5 \n\t" /* uvbuf0[eax+2048]*/\
577 "movq "AV_STRINGIFY(VOF)"(%3, "#index"), %%mm4 \n\t" /* uvbuf1[eax+2048]*/\
578 "paddw %%mm2, %%mm3 \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\
579 "paddw %%mm5, %%mm4 \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\
580 "psrlw $5, %%mm3 \n\t" /*FIXME might overflow*/\
581 "psrlw $5, %%mm4 \n\t" /*FIXME might overflow*/\
582 "psubw "U_OFFSET"("#c"), %%mm3 \n\t" /* (U-128)8*/\
583 "psubw "V_OFFSET"("#c"), %%mm4 \n\t" /* (V-128)8*/\
584 "movq %%mm3, %%mm2 \n\t" /* (U-128)8*/\
585 "movq %%mm4, %%mm5 \n\t" /* (V-128)8*/\
586 "pmulhw "UG_COEFF"("#c"), %%mm3 \n\t"\
587 "pmulhw "VG_COEFF"("#c"), %%mm4 \n\t"\
588 /* mm2=(U-128)8, mm3=ug, mm4=vg mm5=(V-128)8 */\
589 "movq (%0, "#index", 2), %%mm1 \n\t" /*buf0[eax]*/\
590 "movq 8(%0, "#index", 2), %%mm7 \n\t" /*buf0[eax]*/\
591 "psraw $4, %%mm1 \n\t" /* buf0[eax] - buf1[eax] >>4*/\
592 "psraw $4, %%mm7 \n\t" /* buf0[eax] - buf1[eax] >>4*/\
593 "pmulhw "UB_COEFF"("#c"), %%mm2 \n\t"\
594 "pmulhw "VR_COEFF"("#c"), %%mm5 \n\t"\
595 "psubw "Y_OFFSET"("#c"), %%mm1 \n\t" /* 8(Y-16)*/\
596 "psubw "Y_OFFSET"("#c"), %%mm7 \n\t" /* 8(Y-16)*/\
597 "pmulhw "Y_COEFF"("#c"), %%mm1 \n\t"\
598 "pmulhw "Y_COEFF"("#c"), %%mm7 \n\t"\
599 /* mm1= Y1, mm2=ub, mm3=ug, mm4=vg mm5=vr, mm7=Y2 */\
600 "paddw %%mm3, %%mm4 \n\t"\
601 "movq %%mm2, %%mm0 \n\t"\
602 "movq %%mm5, %%mm6 \n\t"\
603 "movq %%mm4, %%mm3 \n\t"\
604 "punpcklwd %%mm2, %%mm2 \n\t"\
605 "punpcklwd %%mm5, %%mm5 \n\t"\
606 "punpcklwd %%mm4, %%mm4 \n\t"\
607 "paddw %%mm1, %%mm2 \n\t"\
608 "paddw %%mm1, %%mm5 \n\t"\
609 "paddw %%mm1, %%mm4 \n\t"\
610 "punpckhwd %%mm0, %%mm0 \n\t"\
611 "punpckhwd %%mm6, %%mm6 \n\t"\
612 "punpckhwd %%mm3, %%mm3 \n\t"\
613 "paddw %%mm7, %%mm0 \n\t"\
614 "paddw %%mm7, %%mm6 \n\t"\
615 "paddw %%mm7, %%mm3 \n\t"\
616 /* mm0=B1, mm2=B2, mm3=G2, mm4=G1, mm5=R1, mm6=R2 */\
617 "packuswb %%mm0, %%mm2 \n\t"\
618 "packuswb %%mm6, %%mm5 \n\t"\
619 "packuswb %%mm3, %%mm4 \n\t"\
621 #define YSCALEYUV2RGB1b(index, c) REAL_YSCALEYUV2RGB1b(index, c)
623 #define REAL_YSCALEYUV2RGB1_ALPHA(index) \
624 "movq (%1, "#index", 2), %%mm7 \n\t" /* abuf0[index ] */\
625 "movq 8(%1, "#index", 2), %%mm1 \n\t" /* abuf0[index+4] */\
626 "psraw $7, %%mm7 \n\t" /* abuf0[index ] >>7 */\
627 "psraw $7, %%mm1 \n\t" /* abuf0[index+4] >>7 */\
628 "packuswb %%mm1, %%mm7 \n\t"
629 #define YSCALEYUV2RGB1_ALPHA(index) REAL_YSCALEYUV2RGB1_ALPHA(index)
631 #define REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) \
632 "movq "#b", "#q2" \n\t" /* B */\
633 "movq "#r", "#t" \n\t" /* R */\
634 "punpcklbw "#g", "#b" \n\t" /* GBGBGBGB 0 */\
635 "punpcklbw "#a", "#r" \n\t" /* ARARARAR 0 */\
636 "punpckhbw "#g", "#q2" \n\t" /* GBGBGBGB 2 */\
637 "punpckhbw "#a", "#t" \n\t" /* ARARARAR 2 */\
638 "movq "#b", "#q0" \n\t" /* GBGBGBGB 0 */\
639 "movq "#q2", "#q3" \n\t" /* GBGBGBGB 2 */\
640 "punpcklwd "#r", "#q0" \n\t" /* ARGBARGB 0 */\
641 "punpckhwd "#r", "#b" \n\t" /* ARGBARGB 1 */\
642 "punpcklwd "#t", "#q2" \n\t" /* ARGBARGB 2 */\
643 "punpckhwd "#t", "#q3" \n\t" /* ARGBARGB 3 */\
645 MOVNTQ( q0, (dst, index, 4))\
646 MOVNTQ( b, 8(dst, index, 4))\
647 MOVNTQ( q2, 16(dst, index, 4))\
648 MOVNTQ( q3, 24(dst, index, 4))\
650 "add $8, "#index" \n\t"\
651 "cmp "#dstw", "#index" \n\t"\
653 #define WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t) REAL_WRITEBGR32(dst, dstw, index, b, g, r, a, q0, q2, q3, t)
655 #define REAL_WRITERGB16(dst, dstw, index) \
656 "pand "MANGLE(bF8)", %%mm2 \n\t" /* B */\
657 "pand "MANGLE(bFC)", %%mm4 \n\t" /* G */\
658 "pand "MANGLE(bF8)", %%mm5 \n\t" /* R */\
659 "psrlq $3, %%mm2 \n\t"\
661 "movq %%mm2, %%mm1 \n\t"\
662 "movq %%mm4, %%mm3 \n\t"\
664 "punpcklbw %%mm7, %%mm3 \n\t"\
665 "punpcklbw %%mm5, %%mm2 \n\t"\
666 "punpckhbw %%mm7, %%mm4 \n\t"\
667 "punpckhbw %%mm5, %%mm1 \n\t"\
669 "psllq $3, %%mm3 \n\t"\
670 "psllq $3, %%mm4 \n\t"\
672 "por %%mm3, %%mm2 \n\t"\
673 "por %%mm4, %%mm1 \n\t"\
675 MOVNTQ(%%mm2, (dst, index, 2))\
676 MOVNTQ(%%mm1, 8(dst, index, 2))\
678 "add $8, "#index" \n\t"\
679 "cmp "#dstw", "#index" \n\t"\
681 #define WRITERGB16(dst, dstw, index) REAL_WRITERGB16(dst, dstw, index)
683 #define REAL_WRITERGB15(dst, dstw, index) \
684 "pand "MANGLE(bF8)", %%mm2 \n\t" /* B */\
685 "pand "MANGLE(bF8)", %%mm4 \n\t" /* G */\
686 "pand "MANGLE(bF8)", %%mm5 \n\t" /* R */\
687 "psrlq $3, %%mm2 \n\t"\
688 "psrlq $1, %%mm5 \n\t"\
690 "movq %%mm2, %%mm1 \n\t"\
691 "movq %%mm4, %%mm3 \n\t"\
693 "punpcklbw %%mm7, %%mm3 \n\t"\
694 "punpcklbw %%mm5, %%mm2 \n\t"\
695 "punpckhbw %%mm7, %%mm4 \n\t"\
696 "punpckhbw %%mm5, %%mm1 \n\t"\
698 "psllq $2, %%mm3 \n\t"\
699 "psllq $2, %%mm4 \n\t"\
701 "por %%mm3, %%mm2 \n\t"\
702 "por %%mm4, %%mm1 \n\t"\
704 MOVNTQ(%%mm2, (dst, index, 2))\
705 MOVNTQ(%%mm1, 8(dst, index, 2))\
707 "add $8, "#index" \n\t"\
708 "cmp "#dstw", "#index" \n\t"\
710 #define WRITERGB15(dst, dstw, index) REAL_WRITERGB15(dst, dstw, index)
712 #define WRITEBGR24OLD(dst, dstw, index) \
713 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
714 "movq %%mm2, %%mm1 \n\t" /* B */\
715 "movq %%mm5, %%mm6 \n\t" /* R */\
716 "punpcklbw %%mm4, %%mm2 \n\t" /* GBGBGBGB 0 */\
717 "punpcklbw %%mm7, %%mm5 \n\t" /* 0R0R0R0R 0 */\
718 "punpckhbw %%mm4, %%mm1 \n\t" /* GBGBGBGB 2 */\
719 "punpckhbw %%mm7, %%mm6 \n\t" /* 0R0R0R0R 2 */\
720 "movq %%mm2, %%mm0 \n\t" /* GBGBGBGB 0 */\
721 "movq %%mm1, %%mm3 \n\t" /* GBGBGBGB 2 */\
722 "punpcklwd %%mm5, %%mm0 \n\t" /* 0RGB0RGB 0 */\
723 "punpckhwd %%mm5, %%mm2 \n\t" /* 0RGB0RGB 1 */\
724 "punpcklwd %%mm6, %%mm1 \n\t" /* 0RGB0RGB 2 */\
725 "punpckhwd %%mm6, %%mm3 \n\t" /* 0RGB0RGB 3 */\
727 "movq %%mm0, %%mm4 \n\t" /* 0RGB0RGB 0 */\
728 "psrlq $8, %%mm0 \n\t" /* 00RGB0RG 0 */\
729 "pand "MANGLE(bm00000111)", %%mm4 \n\t" /* 00000RGB 0 */\
730 "pand "MANGLE(bm11111000)", %%mm0 \n\t" /* 00RGB000 0.5 */\
731 "por %%mm4, %%mm0 \n\t" /* 00RGBRGB 0 */\
732 "movq %%mm2, %%mm4 \n\t" /* 0RGB0RGB 1 */\
733 "psllq $48, %%mm2 \n\t" /* GB000000 1 */\
734 "por %%mm2, %%mm0 \n\t" /* GBRGBRGB 0 */\
736 "movq %%mm4, %%mm2 \n\t" /* 0RGB0RGB 1 */\
737 "psrld $16, %%mm4 \n\t" /* 000R000R 1 */\
738 "psrlq $24, %%mm2 \n\t" /* 0000RGB0 1.5 */\
739 "por %%mm4, %%mm2 \n\t" /* 000RRGBR 1 */\
740 "pand "MANGLE(bm00001111)", %%mm2 \n\t" /* 0000RGBR 1 */\
741 "movq %%mm1, %%mm4 \n\t" /* 0RGB0RGB 2 */\
742 "psrlq $8, %%mm1 \n\t" /* 00RGB0RG 2 */\
743 "pand "MANGLE(bm00000111)", %%mm4 \n\t" /* 00000RGB 2 */\
744 "pand "MANGLE(bm11111000)", %%mm1 \n\t" /* 00RGB000 2.5 */\
745 "por %%mm4, %%mm1 \n\t" /* 00RGBRGB 2 */\
746 "movq %%mm1, %%mm4 \n\t" /* 00RGBRGB 2 */\
747 "psllq $32, %%mm1 \n\t" /* BRGB0000 2 */\
748 "por %%mm1, %%mm2 \n\t" /* BRGBRGBR 1 */\
750 "psrlq $32, %%mm4 \n\t" /* 000000RG 2.5 */\
751 "movq %%mm3, %%mm5 \n\t" /* 0RGB0RGB 3 */\
752 "psrlq $8, %%mm3 \n\t" /* 00RGB0RG 3 */\
753 "pand "MANGLE(bm00000111)", %%mm5 \n\t" /* 00000RGB 3 */\
754 "pand "MANGLE(bm11111000)", %%mm3 \n\t" /* 00RGB000 3.5 */\
755 "por %%mm5, %%mm3 \n\t" /* 00RGBRGB 3 */\
756 "psllq $16, %%mm3 \n\t" /* RGBRGB00 3 */\
757 "por %%mm4, %%mm3 \n\t" /* RGBRGBRG 2.5 */\
759 MOVNTQ(%%mm0, (dst))\
760 MOVNTQ(%%mm2, 8(dst))\
761 MOVNTQ(%%mm3, 16(dst))\
762 "add $24, "#dst" \n\t"\
764 "add $8, "#index" \n\t"\
765 "cmp "#dstw", "#index" \n\t"\
768 #define WRITEBGR24MMX(dst, dstw, index) \
769 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
770 "movq %%mm2, %%mm1 \n\t" /* B */\
771 "movq %%mm5, %%mm6 \n\t" /* R */\
772 "punpcklbw %%mm4, %%mm2 \n\t" /* GBGBGBGB 0 */\
773 "punpcklbw %%mm7, %%mm5 \n\t" /* 0R0R0R0R 0 */\
774 "punpckhbw %%mm4, %%mm1 \n\t" /* GBGBGBGB 2 */\
775 "punpckhbw %%mm7, %%mm6 \n\t" /* 0R0R0R0R 2 */\
776 "movq %%mm2, %%mm0 \n\t" /* GBGBGBGB 0 */\
777 "movq %%mm1, %%mm3 \n\t" /* GBGBGBGB 2 */\
778 "punpcklwd %%mm5, %%mm0 \n\t" /* 0RGB0RGB 0 */\
779 "punpckhwd %%mm5, %%mm2 \n\t" /* 0RGB0RGB 1 */\
780 "punpcklwd %%mm6, %%mm1 \n\t" /* 0RGB0RGB 2 */\
781 "punpckhwd %%mm6, %%mm3 \n\t" /* 0RGB0RGB 3 */\
783 "movq %%mm0, %%mm4 \n\t" /* 0RGB0RGB 0 */\
784 "movq %%mm2, %%mm6 \n\t" /* 0RGB0RGB 1 */\
785 "movq %%mm1, %%mm5 \n\t" /* 0RGB0RGB 2 */\
786 "movq %%mm3, %%mm7 \n\t" /* 0RGB0RGB 3 */\
788 "psllq $40, %%mm0 \n\t" /* RGB00000 0 */\
789 "psllq $40, %%mm2 \n\t" /* RGB00000 1 */\
790 "psllq $40, %%mm1 \n\t" /* RGB00000 2 */\
791 "psllq $40, %%mm3 \n\t" /* RGB00000 3 */\
793 "punpckhdq %%mm4, %%mm0 \n\t" /* 0RGBRGB0 0 */\
794 "punpckhdq %%mm6, %%mm2 \n\t" /* 0RGBRGB0 1 */\
795 "punpckhdq %%mm5, %%mm1 \n\t" /* 0RGBRGB0 2 */\
796 "punpckhdq %%mm7, %%mm3 \n\t" /* 0RGBRGB0 3 */\
798 "psrlq $8, %%mm0 \n\t" /* 00RGBRGB 0 */\
799 "movq %%mm2, %%mm6 \n\t" /* 0RGBRGB0 1 */\
800 "psllq $40, %%mm2 \n\t" /* GB000000 1 */\
801 "por %%mm2, %%mm0 \n\t" /* GBRGBRGB 0 */\
802 MOVNTQ(%%mm0, (dst))\
804 "psrlq $24, %%mm6 \n\t" /* 0000RGBR 1 */\
805 "movq %%mm1, %%mm5 \n\t" /* 0RGBRGB0 2 */\
806 "psllq $24, %%mm1 \n\t" /* BRGB0000 2 */\
807 "por %%mm1, %%mm6 \n\t" /* BRGBRGBR 1 */\
808 MOVNTQ(%%mm6, 8(dst))\
810 "psrlq $40, %%mm5 \n\t" /* 000000RG 2 */\
811 "psllq $8, %%mm3 \n\t" /* RGBRGB00 3 */\
812 "por %%mm3, %%mm5 \n\t" /* RGBRGBRG 2 */\
813 MOVNTQ(%%mm5, 16(dst))\
815 "add $24, "#dst" \n\t"\
817 "add $8, "#index" \n\t"\
818 "cmp "#dstw", "#index" \n\t"\
821 #define WRITEBGR24MMX2(dst, dstw, index) \
822 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */\
823 "movq "MANGLE(ff_M24A)", %%mm0 \n\t"\
824 "movq "MANGLE(ff_M24C)", %%mm7 \n\t"\
825 "pshufw $0x50, %%mm2, %%mm1 \n\t" /* B3 B2 B3 B2 B1 B0 B1 B0 */\
826 "pshufw $0x50, %%mm4, %%mm3 \n\t" /* G3 G2 G3 G2 G1 G0 G1 G0 */\
827 "pshufw $0x00, %%mm5, %%mm6 \n\t" /* R1 R0 R1 R0 R1 R0 R1 R0 */\
829 "pand %%mm0, %%mm1 \n\t" /* B2 B1 B0 */\
830 "pand %%mm0, %%mm3 \n\t" /* G2 G1 G0 */\
831 "pand %%mm7, %%mm6 \n\t" /* R1 R0 */\
833 "psllq $8, %%mm3 \n\t" /* G2 G1 G0 */\
834 "por %%mm1, %%mm6 \n\t"\
835 "por %%mm3, %%mm6 \n\t"\
836 MOVNTQ(%%mm6, (dst))\
838 "psrlq $8, %%mm4 \n\t" /* 00 G7 G6 G5 G4 G3 G2 G1 */\
839 "pshufw $0xA5, %%mm2, %%mm1 \n\t" /* B5 B4 B5 B4 B3 B2 B3 B2 */\
840 "pshufw $0x55, %%mm4, %%mm3 \n\t" /* G4 G3 G4 G3 G4 G3 G4 G3 */\
841 "pshufw $0xA5, %%mm5, %%mm6 \n\t" /* R5 R4 R5 R4 R3 R2 R3 R2 */\
843 "pand "MANGLE(ff_M24B)", %%mm1 \n\t" /* B5 B4 B3 */\
844 "pand %%mm7, %%mm3 \n\t" /* G4 G3 */\
845 "pand %%mm0, %%mm6 \n\t" /* R4 R3 R2 */\
847 "por %%mm1, %%mm3 \n\t" /* B5 G4 B4 G3 B3 */\
848 "por %%mm3, %%mm6 \n\t"\
849 MOVNTQ(%%mm6, 8(dst))\
851 "pshufw $0xFF, %%mm2, %%mm1 \n\t" /* B7 B6 B7 B6 B7 B6 B6 B7 */\
852 "pshufw $0xFA, %%mm4, %%mm3 \n\t" /* 00 G7 00 G7 G6 G5 G6 G5 */\
853 "pshufw $0xFA, %%mm5, %%mm6 \n\t" /* R7 R6 R7 R6 R5 R4 R5 R4 */\
855 "pand %%mm7, %%mm1 \n\t" /* B7 B6 */\
856 "pand %%mm0, %%mm3 \n\t" /* G7 G6 G5 */\
857 "pand "MANGLE(ff_M24B)", %%mm6 \n\t" /* R7 R6 R5 */\
859 "por %%mm1, %%mm3 \n\t"\
860 "por %%mm3, %%mm6 \n\t"\
861 MOVNTQ(%%mm6, 16(dst))\
863 "add $24, "#dst" \n\t"\
865 "add $8, "#index" \n\t"\
866 "cmp "#dstw", "#index" \n\t"\
869 #if COMPILE_TEMPLATE_MMX2
871 #define WRITEBGR24(dst, dstw, index) WRITEBGR24MMX2(dst, dstw, index)
874 #define WRITEBGR24(dst, dstw, index) WRITEBGR24MMX(dst, dstw, index)
877 #define REAL_WRITEYUY2(dst, dstw, index) \
878 "packuswb %%mm3, %%mm3 \n\t"\
879 "packuswb %%mm4, %%mm4 \n\t"\
880 "packuswb %%mm7, %%mm1 \n\t"\
881 "punpcklbw %%mm4, %%mm3 \n\t"\
882 "movq %%mm1, %%mm7 \n\t"\
883 "punpcklbw %%mm3, %%mm1 \n\t"\
884 "punpckhbw %%mm3, %%mm7 \n\t"\
886 MOVNTQ(%%mm1, (dst, index, 2))\
887 MOVNTQ(%%mm7, 8(dst, index, 2))\
889 "add $8, "#index" \n\t"\
890 "cmp "#dstw", "#index" \n\t"\
892 #define WRITEYUY2(dst, dstw, index) REAL_WRITEYUY2(dst, dstw, index)
895 static inline void RENAME(yuv2yuvX)(SwsContext *c, const int16_t *lumFilter, const int16_t **lumSrc, int lumFilterSize,
896 const int16_t *chrFilter, const int16_t **chrSrc, int chrFilterSize, const int16_t **alpSrc,
897 uint8_t *dest, uint8_t *uDest, uint8_t *vDest, uint8_t *aDest, long dstW, long chrDstW)
899 if(!(c->flags & SWS_BITEXACT)) {
900 if (c->flags & SWS_ACCURATE_RND) {
902 YSCALEYUV2YV12X_ACCURATE( "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
903 YSCALEYUV2YV12X_ACCURATE(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
905 if (CONFIG_SWSCALE_ALPHA && aDest) {
906 YSCALEYUV2YV12X_ACCURATE( "0", ALP_MMX_FILTER_OFFSET, aDest, dstW)
909 YSCALEYUV2YV12X_ACCURATE("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
912 YSCALEYUV2YV12X( "0", CHR_MMX_FILTER_OFFSET, uDest, chrDstW)
913 YSCALEYUV2YV12X(AV_STRINGIFY(VOF), CHR_MMX_FILTER_OFFSET, vDest, chrDstW)
915 if (CONFIG_SWSCALE_ALPHA && aDest) {
916 YSCALEYUV2YV12X( "0", ALP_MMX_FILTER_OFFSET, aDest, dstW)
919 YSCALEYUV2YV12X("0", LUM_MMX_FILTER_OFFSET, dest, dstW)
923 yuv2yuvXinC(lumFilter, lumSrc, lumFilterSize,
924 chrFilter, chrSrc, chrFilterSize,
925 alpSrc, dest, uDest, vDest, aDest, dstW, chrDstW);
928 static inline void RENAME(yuv2yuv1)(SwsContext *c, const int16_t *lumSrc, const int16_t *chrSrc, const int16_t *alpSrc,
929 uint8_t *dest, uint8_t *uDest, uint8_t *vDest, uint8_t *aDest, long dstW, long chrDstW)
931 if(!(c->flags & SWS_BITEXACT)) {
933 const uint8_t *src[4]= {alpSrc + dstW, lumSrc + dstW, chrSrc + chrDstW, chrSrc + VOFW + chrDstW};
934 uint8_t *dst[4]= {aDest, dest, uDest, vDest};
935 x86_reg counter[4]= {dstW, dstW, chrDstW, chrDstW};
937 if (c->flags & SWS_ACCURATE_RND) {
941 YSCALEYUV2YV121_ACCURATE
942 :: "r" (src[p]), "r" (dst[p] + counter[p]),
953 :: "r" (src[p]), "r" (dst[p] + counter[p]),
962 yuv2yuv1_c(c, lumSrc, chrSrc, alpSrc, dest, uDest, vDest, aDest,
968 * vertical scale YV12 to RGB
970 static inline void RENAME(yuv2packedX)(SwsContext *c, const int16_t *lumFilter, const int16_t **lumSrc, int lumFilterSize,
971 const int16_t *chrFilter, const int16_t **chrSrc, int chrFilterSize,
972 const int16_t **alpSrc, uint8_t *dest, long dstW, long dstY)
975 x86_reg dstW_reg = dstW;
976 if(!(c->flags & SWS_BITEXACT)) {
977 if (c->flags & SWS_ACCURATE_RND) {
978 switch(c->dstFormat) {
980 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
981 YSCALEYUV2PACKEDX_ACCURATE
983 "movq %%mm2, "U_TEMP"(%0) \n\t"
984 "movq %%mm4, "V_TEMP"(%0) \n\t"
985 "movq %%mm5, "Y_TEMP"(%0) \n\t"
986 YSCALEYUV2PACKEDX_ACCURATE_YA(ALP_MMX_FILTER_OFFSET)
987 "movq "Y_TEMP"(%0), %%mm5 \n\t"
988 "psraw $3, %%mm1 \n\t"
989 "psraw $3, %%mm7 \n\t"
990 "packuswb %%mm7, %%mm1 \n\t"
991 WRITEBGR32(%4, %5, %%REGa, %%mm3, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm2, %%mm6)
993 YSCALEYUV2PACKEDX_END
995 YSCALEYUV2PACKEDX_ACCURATE
997 "pcmpeqd %%mm7, %%mm7 \n\t"
998 WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1000 YSCALEYUV2PACKEDX_END
1004 YSCALEYUV2PACKEDX_ACCURATE
1006 "pxor %%mm7, %%mm7 \n\t"
1007 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c"\n\t" //FIXME optimize
1008 "add %4, %%"REG_c" \n\t"
1009 WRITEBGR24(%%REGc, %5, %%REGa)
1012 :: "r" (&c->redDither),
1013 "m" (dummy), "m" (dummy), "m" (dummy),
1014 "r" (dest), "m" (dstW_reg)
1015 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
1018 case PIX_FMT_RGB555:
1019 YSCALEYUV2PACKEDX_ACCURATE
1021 "pxor %%mm7, %%mm7 \n\t"
1022 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1024 "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
1025 "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
1026 "paddusb "RED_DITHER"(%0), %%mm5\n\t"
1029 WRITERGB15(%4, %5, %%REGa)
1030 YSCALEYUV2PACKEDX_END
1032 case PIX_FMT_RGB565:
1033 YSCALEYUV2PACKEDX_ACCURATE
1035 "pxor %%mm7, %%mm7 \n\t"
1036 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1038 "paddusb "BLUE_DITHER"(%0), %%mm2\n\t"
1039 "paddusb "GREEN_DITHER"(%0), %%mm4\n\t"
1040 "paddusb "RED_DITHER"(%0), %%mm5\n\t"
1043 WRITERGB16(%4, %5, %%REGa)
1044 YSCALEYUV2PACKEDX_END
1046 case PIX_FMT_YUYV422:
1047 YSCALEYUV2PACKEDX_ACCURATE
1048 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1050 "psraw $3, %%mm3 \n\t"
1051 "psraw $3, %%mm4 \n\t"
1052 "psraw $3, %%mm1 \n\t"
1053 "psraw $3, %%mm7 \n\t"
1054 WRITEYUY2(%4, %5, %%REGa)
1055 YSCALEYUV2PACKEDX_END
1059 switch(c->dstFormat) {
1061 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1064 YSCALEYUV2PACKEDX_YA(ALP_MMX_FILTER_OFFSET, %%mm0, %%mm3, %%mm6, %%mm1, %%mm7)
1065 "psraw $3, %%mm1 \n\t"
1066 "psraw $3, %%mm7 \n\t"
1067 "packuswb %%mm7, %%mm1 \n\t"
1068 WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1069 YSCALEYUV2PACKEDX_END
1073 "pcmpeqd %%mm7, %%mm7 \n\t"
1074 WRITEBGR32(%4, %5, %%REGa, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1075 YSCALEYUV2PACKEDX_END
1081 "pxor %%mm7, %%mm7 \n\t"
1082 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_c" \n\t" //FIXME optimize
1083 "add %4, %%"REG_c" \n\t"
1084 WRITEBGR24(%%REGc, %5, %%REGa)
1086 :: "r" (&c->redDither),
1087 "m" (dummy), "m" (dummy), "m" (dummy),
1088 "r" (dest), "m" (dstW_reg)
1089 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S
1092 case PIX_FMT_RGB555:
1095 "pxor %%mm7, %%mm7 \n\t"
1096 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1098 "paddusb "BLUE_DITHER"(%0), %%mm2 \n\t"
1099 "paddusb "GREEN_DITHER"(%0), %%mm4 \n\t"
1100 "paddusb "RED_DITHER"(%0), %%mm5 \n\t"
1103 WRITERGB15(%4, %5, %%REGa)
1104 YSCALEYUV2PACKEDX_END
1106 case PIX_FMT_RGB565:
1109 "pxor %%mm7, %%mm7 \n\t"
1110 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1112 "paddusb "BLUE_DITHER"(%0), %%mm2 \n\t"
1113 "paddusb "GREEN_DITHER"(%0), %%mm4 \n\t"
1114 "paddusb "RED_DITHER"(%0), %%mm5 \n\t"
1117 WRITERGB16(%4, %5, %%REGa)
1118 YSCALEYUV2PACKEDX_END
1120 case PIX_FMT_YUYV422:
1122 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1124 "psraw $3, %%mm3 \n\t"
1125 "psraw $3, %%mm4 \n\t"
1126 "psraw $3, %%mm1 \n\t"
1127 "psraw $3, %%mm7 \n\t"
1128 WRITEYUY2(%4, %5, %%REGa)
1129 YSCALEYUV2PACKEDX_END
1134 yuv2packedXinC(c, lumFilter, lumSrc, lumFilterSize,
1135 chrFilter, chrSrc, chrFilterSize,
1136 alpSrc, dest, dstW, dstY);
1140 * vertical bilinear scale YV12 to RGB
1142 static inline void RENAME(yuv2packed2)(SwsContext *c, const uint16_t *buf0, const uint16_t *buf1, const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1143 const uint16_t *abuf0, const uint16_t *abuf1, uint8_t *dest, int dstW, int yalpha, int uvalpha, int y)
1145 if(!(c->flags & SWS_BITEXACT)) {
1146 switch(c->dstFormat) {
1147 //Note 8280 == DSTW_OFFSET but the preprocessor can't handle that there :(
1149 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1152 YSCALEYUV2RGB(%%r8, %5)
1153 YSCALEYUV2RGB_YA(%%r8, %5, %6, %7)
1154 "psraw $3, %%mm1 \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1155 "psraw $3, %%mm7 \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1156 "packuswb %%mm7, %%mm1 \n\t"
1157 WRITEBGR32(%4, 8280(%5), %%r8, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1159 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "r" (dest),
1161 ,"r" (abuf0), "r" (abuf1)
1165 *(const uint16_t **)(&c->u_temp)=abuf0;
1166 *(const uint16_t **)(&c->v_temp)=abuf1;
1168 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1169 "mov %4, %%"REG_b" \n\t"
1170 "push %%"REG_BP" \n\t"
1171 YSCALEYUV2RGB(%%REGBP, %5)
1174 "mov "U_TEMP"(%5), %0 \n\t"
1175 "mov "V_TEMP"(%5), %1 \n\t"
1176 YSCALEYUV2RGB_YA(%%REGBP, %5, %0, %1)
1177 "psraw $3, %%mm1 \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1178 "psraw $3, %%mm7 \n\t" /* abuf0[eax] - abuf1[eax] >>7*/
1179 "packuswb %%mm7, %%mm1 \n\t"
1182 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm1, %%mm0, %%mm7, %%mm3, %%mm6)
1183 "pop %%"REG_BP" \n\t"
1184 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1186 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1192 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1193 "mov %4, %%"REG_b" \n\t"
1194 "push %%"REG_BP" \n\t"
1195 YSCALEYUV2RGB(%%REGBP, %5)
1196 "pcmpeqd %%mm7, %%mm7 \n\t"
1197 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1198 "pop %%"REG_BP" \n\t"
1199 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1201 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1208 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1209 "mov %4, %%"REG_b" \n\t"
1210 "push %%"REG_BP" \n\t"
1211 YSCALEYUV2RGB(%%REGBP, %5)
1212 "pxor %%mm7, %%mm7 \n\t"
1213 WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1214 "pop %%"REG_BP" \n\t"
1215 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1216 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1220 case PIX_FMT_RGB555:
1222 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1223 "mov %4, %%"REG_b" \n\t"
1224 "push %%"REG_BP" \n\t"
1225 YSCALEYUV2RGB(%%REGBP, %5)
1226 "pxor %%mm7, %%mm7 \n\t"
1227 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1229 "paddusb "BLUE_DITHER"(%5), %%mm2 \n\t"
1230 "paddusb "GREEN_DITHER"(%5), %%mm4 \n\t"
1231 "paddusb "RED_DITHER"(%5), %%mm5 \n\t"
1234 WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1235 "pop %%"REG_BP" \n\t"
1236 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1238 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1242 case PIX_FMT_RGB565:
1244 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1245 "mov %4, %%"REG_b" \n\t"
1246 "push %%"REG_BP" \n\t"
1247 YSCALEYUV2RGB(%%REGBP, %5)
1248 "pxor %%mm7, %%mm7 \n\t"
1249 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1251 "paddusb "BLUE_DITHER"(%5), %%mm2 \n\t"
1252 "paddusb "GREEN_DITHER"(%5), %%mm4 \n\t"
1253 "paddusb "RED_DITHER"(%5), %%mm5 \n\t"
1256 WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1257 "pop %%"REG_BP" \n\t"
1258 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1259 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1263 case PIX_FMT_YUYV422:
1265 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1266 "mov %4, %%"REG_b" \n\t"
1267 "push %%"REG_BP" \n\t"
1268 YSCALEYUV2PACKED(%%REGBP, %5)
1269 WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1270 "pop %%"REG_BP" \n\t"
1271 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1272 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1279 yuv2packed2_c(c, buf0, buf1, uvbuf0, uvbuf1, abuf0, abuf1,
1280 dest, dstW, yalpha, uvalpha, y);
1284 * YV12 to RGB without scaling or interpolating
1286 static inline void RENAME(yuv2packed1)(SwsContext *c, const uint16_t *buf0, const uint16_t *uvbuf0, const uint16_t *uvbuf1,
1287 const uint16_t *abuf0, uint8_t *dest, int dstW, int uvalpha, enum PixelFormat dstFormat, int flags, int y)
1289 if(!(flags & SWS_BITEXACT)) {
1290 const uint16_t *buf1= buf0; //FIXME needed for RGB1/BGR1
1292 if (flags&SWS_FULL_CHR_H_INT) {
1293 c->yuv2packed2(c, buf0, buf0, uvbuf0, uvbuf1, abuf0, abuf0, dest, dstW, 0, uvalpha, y);
1297 if (uvalpha < 2048) { // note this is not correct (shifts chrominance by 0.5 pixels) but it is a bit faster
1300 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1302 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1303 "mov %4, %%"REG_b" \n\t"
1304 "push %%"REG_BP" \n\t"
1305 YSCALEYUV2RGB1(%%REGBP, %5)
1306 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1307 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1308 "pop %%"REG_BP" \n\t"
1309 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1311 :: "c" (buf0), "d" (abuf0), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1316 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1317 "mov %4, %%"REG_b" \n\t"
1318 "push %%"REG_BP" \n\t"
1319 YSCALEYUV2RGB1(%%REGBP, %5)
1320 "pcmpeqd %%mm7, %%mm7 \n\t"
1321 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1322 "pop %%"REG_BP" \n\t"
1323 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1325 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1332 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1333 "mov %4, %%"REG_b" \n\t"
1334 "push %%"REG_BP" \n\t"
1335 YSCALEYUV2RGB1(%%REGBP, %5)
1336 "pxor %%mm7, %%mm7 \n\t"
1337 WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1338 "pop %%"REG_BP" \n\t"
1339 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1341 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1345 case PIX_FMT_RGB555:
1347 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1348 "mov %4, %%"REG_b" \n\t"
1349 "push %%"REG_BP" \n\t"
1350 YSCALEYUV2RGB1(%%REGBP, %5)
1351 "pxor %%mm7, %%mm7 \n\t"
1352 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1354 "paddusb "BLUE_DITHER"(%5), %%mm2 \n\t"
1355 "paddusb "GREEN_DITHER"(%5), %%mm4 \n\t"
1356 "paddusb "RED_DITHER"(%5), %%mm5 \n\t"
1358 WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1359 "pop %%"REG_BP" \n\t"
1360 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1362 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1366 case PIX_FMT_RGB565:
1368 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1369 "mov %4, %%"REG_b" \n\t"
1370 "push %%"REG_BP" \n\t"
1371 YSCALEYUV2RGB1(%%REGBP, %5)
1372 "pxor %%mm7, %%mm7 \n\t"
1373 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1375 "paddusb "BLUE_DITHER"(%5), %%mm2 \n\t"
1376 "paddusb "GREEN_DITHER"(%5), %%mm4 \n\t"
1377 "paddusb "RED_DITHER"(%5), %%mm5 \n\t"
1380 WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1381 "pop %%"REG_BP" \n\t"
1382 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1384 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1388 case PIX_FMT_YUYV422:
1390 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1391 "mov %4, %%"REG_b" \n\t"
1392 "push %%"REG_BP" \n\t"
1393 YSCALEYUV2PACKED1(%%REGBP, %5)
1394 WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1395 "pop %%"REG_BP" \n\t"
1396 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1398 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1406 if (CONFIG_SWSCALE_ALPHA && c->alpPixBuf) {
1408 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1409 "mov %4, %%"REG_b" \n\t"
1410 "push %%"REG_BP" \n\t"
1411 YSCALEYUV2RGB1b(%%REGBP, %5)
1412 YSCALEYUV2RGB1_ALPHA(%%REGBP)
1413 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1414 "pop %%"REG_BP" \n\t"
1415 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1417 :: "c" (buf0), "d" (abuf0), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1422 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1423 "mov %4, %%"REG_b" \n\t"
1424 "push %%"REG_BP" \n\t"
1425 YSCALEYUV2RGB1b(%%REGBP, %5)
1426 "pcmpeqd %%mm7, %%mm7 \n\t"
1427 WRITEBGR32(%%REGb, 8280(%5), %%REGBP, %%mm2, %%mm4, %%mm5, %%mm7, %%mm0, %%mm1, %%mm3, %%mm6)
1428 "pop %%"REG_BP" \n\t"
1429 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1431 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1438 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1439 "mov %4, %%"REG_b" \n\t"
1440 "push %%"REG_BP" \n\t"
1441 YSCALEYUV2RGB1b(%%REGBP, %5)
1442 "pxor %%mm7, %%mm7 \n\t"
1443 WRITEBGR24(%%REGb, 8280(%5), %%REGBP)
1444 "pop %%"REG_BP" \n\t"
1445 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1447 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1451 case PIX_FMT_RGB555:
1453 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1454 "mov %4, %%"REG_b" \n\t"
1455 "push %%"REG_BP" \n\t"
1456 YSCALEYUV2RGB1b(%%REGBP, %5)
1457 "pxor %%mm7, %%mm7 \n\t"
1458 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1460 "paddusb "BLUE_DITHER"(%5), %%mm2 \n\t"
1461 "paddusb "GREEN_DITHER"(%5), %%mm4 \n\t"
1462 "paddusb "RED_DITHER"(%5), %%mm5 \n\t"
1464 WRITERGB15(%%REGb, 8280(%5), %%REGBP)
1465 "pop %%"REG_BP" \n\t"
1466 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1468 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1472 case PIX_FMT_RGB565:
1474 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1475 "mov %4, %%"REG_b" \n\t"
1476 "push %%"REG_BP" \n\t"
1477 YSCALEYUV2RGB1b(%%REGBP, %5)
1478 "pxor %%mm7, %%mm7 \n\t"
1479 /* mm2=B, %%mm4=G, %%mm5=R, %%mm7=0 */
1481 "paddusb "BLUE_DITHER"(%5), %%mm2 \n\t"
1482 "paddusb "GREEN_DITHER"(%5), %%mm4 \n\t"
1483 "paddusb "RED_DITHER"(%5), %%mm5 \n\t"
1486 WRITERGB16(%%REGb, 8280(%5), %%REGBP)
1487 "pop %%"REG_BP" \n\t"
1488 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1490 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1494 case PIX_FMT_YUYV422:
1496 "mov %%"REG_b", "ESP_OFFSET"(%5) \n\t"
1497 "mov %4, %%"REG_b" \n\t"
1498 "push %%"REG_BP" \n\t"
1499 YSCALEYUV2PACKED1b(%%REGBP, %5)
1500 WRITEYUY2(%%REGb, 8280(%5), %%REGBP)
1501 "pop %%"REG_BP" \n\t"
1502 "mov "ESP_OFFSET"(%5), %%"REG_b" \n\t"
1504 :: "c" (buf0), "d" (buf1), "S" (uvbuf0), "D" (uvbuf1), "m" (dest),
1511 yuv2packed1_c(c, buf0, uvbuf0, uvbuf1, abuf0, dest,
1512 dstW, uvalpha, dstFormat, flags, y);
1515 //FIXME yuy2* can read up to 7 samples too much
1517 static inline void RENAME(yuy2ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1520 "movq "MANGLE(bm01010101)", %%mm2 \n\t"
1521 "mov %0, %%"REG_a" \n\t"
1523 "movq (%1, %%"REG_a",2), %%mm0 \n\t"
1524 "movq 8(%1, %%"REG_a",2), %%mm1 \n\t"
1525 "pand %%mm2, %%mm0 \n\t"
1526 "pand %%mm2, %%mm1 \n\t"
1527 "packuswb %%mm1, %%mm0 \n\t"
1528 "movq %%mm0, (%2, %%"REG_a") \n\t"
1529 "add $8, %%"REG_a" \n\t"
1531 : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1536 static inline void RENAME(yuy2ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1539 "movq "MANGLE(bm01010101)", %%mm4 \n\t"
1540 "mov %0, %%"REG_a" \n\t"
1542 "movq (%1, %%"REG_a",4), %%mm0 \n\t"
1543 "movq 8(%1, %%"REG_a",4), %%mm1 \n\t"
1544 "psrlw $8, %%mm0 \n\t"
1545 "psrlw $8, %%mm1 \n\t"
1546 "packuswb %%mm1, %%mm0 \n\t"
1547 "movq %%mm0, %%mm1 \n\t"
1548 "psrlw $8, %%mm0 \n\t"
1549 "pand %%mm4, %%mm1 \n\t"
1550 "packuswb %%mm0, %%mm0 \n\t"
1551 "packuswb %%mm1, %%mm1 \n\t"
1552 "movd %%mm0, (%3, %%"REG_a") \n\t"
1553 "movd %%mm1, (%2, %%"REG_a") \n\t"
1554 "add $4, %%"REG_a" \n\t"
1556 : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1559 assert(src1 == src2);
1562 static inline void RENAME(LEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1565 "mov %0, %%"REG_a" \n\t"
1567 "movq (%1, %%"REG_a",2), %%mm0 \n\t"
1568 "movq 8(%1, %%"REG_a",2), %%mm1 \n\t"
1569 "movq (%2, %%"REG_a",2), %%mm2 \n\t"
1570 "movq 8(%2, %%"REG_a",2), %%mm3 \n\t"
1571 "psrlw $8, %%mm0 \n\t"
1572 "psrlw $8, %%mm1 \n\t"
1573 "psrlw $8, %%mm2 \n\t"
1574 "psrlw $8, %%mm3 \n\t"
1575 "packuswb %%mm1, %%mm0 \n\t"
1576 "packuswb %%mm3, %%mm2 \n\t"
1577 "movq %%mm0, (%3, %%"REG_a") \n\t"
1578 "movq %%mm2, (%4, %%"REG_a") \n\t"
1579 "add $8, %%"REG_a" \n\t"
1581 : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1586 /* This is almost identical to the previous, end exists only because
1587 * yuy2ToY/UV)(dst, src+1, ...) would have 100% unaligned accesses. */
1588 static inline void RENAME(uyvyToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1591 "mov %0, %%"REG_a" \n\t"
1593 "movq (%1, %%"REG_a",2), %%mm0 \n\t"
1594 "movq 8(%1, %%"REG_a",2), %%mm1 \n\t"
1595 "psrlw $8, %%mm0 \n\t"
1596 "psrlw $8, %%mm1 \n\t"
1597 "packuswb %%mm1, %%mm0 \n\t"
1598 "movq %%mm0, (%2, %%"REG_a") \n\t"
1599 "add $8, %%"REG_a" \n\t"
1601 : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst+width)
1606 static inline void RENAME(uyvyToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1609 "movq "MANGLE(bm01010101)", %%mm4 \n\t"
1610 "mov %0, %%"REG_a" \n\t"
1612 "movq (%1, %%"REG_a",4), %%mm0 \n\t"
1613 "movq 8(%1, %%"REG_a",4), %%mm1 \n\t"
1614 "pand %%mm4, %%mm0 \n\t"
1615 "pand %%mm4, %%mm1 \n\t"
1616 "packuswb %%mm1, %%mm0 \n\t"
1617 "movq %%mm0, %%mm1 \n\t"
1618 "psrlw $8, %%mm0 \n\t"
1619 "pand %%mm4, %%mm1 \n\t"
1620 "packuswb %%mm0, %%mm0 \n\t"
1621 "packuswb %%mm1, %%mm1 \n\t"
1622 "movd %%mm0, (%3, %%"REG_a") \n\t"
1623 "movd %%mm1, (%2, %%"REG_a") \n\t"
1624 "add $4, %%"REG_a" \n\t"
1626 : : "g" ((x86_reg)-width), "r" (src1+width*4), "r" (dstU+width), "r" (dstV+width)
1629 assert(src1 == src2);
1632 static inline void RENAME(BEToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1635 "movq "MANGLE(bm01010101)", %%mm4 \n\t"
1636 "mov %0, %%"REG_a" \n\t"
1638 "movq (%1, %%"REG_a",2), %%mm0 \n\t"
1639 "movq 8(%1, %%"REG_a",2), %%mm1 \n\t"
1640 "movq (%2, %%"REG_a",2), %%mm2 \n\t"
1641 "movq 8(%2, %%"REG_a",2), %%mm3 \n\t"
1642 "pand %%mm4, %%mm0 \n\t"
1643 "pand %%mm4, %%mm1 \n\t"
1644 "pand %%mm4, %%mm2 \n\t"
1645 "pand %%mm4, %%mm3 \n\t"
1646 "packuswb %%mm1, %%mm0 \n\t"
1647 "packuswb %%mm3, %%mm2 \n\t"
1648 "movq %%mm0, (%3, %%"REG_a") \n\t"
1649 "movq %%mm2, (%4, %%"REG_a") \n\t"
1650 "add $8, %%"REG_a" \n\t"
1652 : : "g" ((x86_reg)-width), "r" (src1+width*2), "r" (src2+width*2), "r" (dstU+width), "r" (dstV+width)
1657 static inline void RENAME(nvXXtoUV)(uint8_t *dst1, uint8_t *dst2,
1658 const uint8_t *src, long width)
1661 "movq "MANGLE(bm01010101)", %%mm4 \n\t"
1662 "mov %0, %%"REG_a" \n\t"
1664 "movq (%1, %%"REG_a",2), %%mm0 \n\t"
1665 "movq 8(%1, %%"REG_a",2), %%mm1 \n\t"
1666 "movq %%mm0, %%mm2 \n\t"
1667 "movq %%mm1, %%mm3 \n\t"
1668 "pand %%mm4, %%mm0 \n\t"
1669 "pand %%mm4, %%mm1 \n\t"
1670 "psrlw $8, %%mm2 \n\t"
1671 "psrlw $8, %%mm3 \n\t"
1672 "packuswb %%mm1, %%mm0 \n\t"
1673 "packuswb %%mm3, %%mm2 \n\t"
1674 "movq %%mm0, (%2, %%"REG_a") \n\t"
1675 "movq %%mm2, (%3, %%"REG_a") \n\t"
1676 "add $8, %%"REG_a" \n\t"
1678 : : "g" ((x86_reg)-width), "r" (src+width*2), "r" (dst1+width), "r" (dst2+width)
1683 static inline void RENAME(nv12ToUV)(uint8_t *dstU, uint8_t *dstV,
1684 const uint8_t *src1, const uint8_t *src2,
1685 long width, uint32_t *unused)
1687 RENAME(nvXXtoUV)(dstU, dstV, src1, width);
1690 static inline void RENAME(nv21ToUV)(uint8_t *dstU, uint8_t *dstV,
1691 const uint8_t *src1, const uint8_t *src2,
1692 long width, uint32_t *unused)
1694 RENAME(nvXXtoUV)(dstV, dstU, src1, width);
1697 static inline void RENAME(bgr24ToY_mmx)(uint8_t *dst, const uint8_t *src, long width, enum PixelFormat srcFormat)
1700 if(srcFormat == PIX_FMT_BGR24) {
1702 "movq "MANGLE(ff_bgr24toY1Coeff)", %%mm5 \n\t"
1703 "movq "MANGLE(ff_bgr24toY2Coeff)", %%mm6 \n\t"
1708 "movq "MANGLE(ff_rgb24toY1Coeff)", %%mm5 \n\t"
1709 "movq "MANGLE(ff_rgb24toY2Coeff)", %%mm6 \n\t"
1715 "movq "MANGLE(ff_bgr24toYOffset)", %%mm4 \n\t"
1716 "mov %2, %%"REG_a" \n\t"
1717 "pxor %%mm7, %%mm7 \n\t"
1719 PREFETCH" 64(%0) \n\t"
1720 "movd (%0), %%mm0 \n\t"
1721 "movd 2(%0), %%mm1 \n\t"
1722 "movd 6(%0), %%mm2 \n\t"
1723 "movd 8(%0), %%mm3 \n\t"
1725 "punpcklbw %%mm7, %%mm0 \n\t"
1726 "punpcklbw %%mm7, %%mm1 \n\t"
1727 "punpcklbw %%mm7, %%mm2 \n\t"
1728 "punpcklbw %%mm7, %%mm3 \n\t"
1729 "pmaddwd %%mm5, %%mm0 \n\t"
1730 "pmaddwd %%mm6, %%mm1 \n\t"
1731 "pmaddwd %%mm5, %%mm2 \n\t"
1732 "pmaddwd %%mm6, %%mm3 \n\t"
1733 "paddd %%mm1, %%mm0 \n\t"
1734 "paddd %%mm3, %%mm2 \n\t"
1735 "paddd %%mm4, %%mm0 \n\t"
1736 "paddd %%mm4, %%mm2 \n\t"
1737 "psrad $15, %%mm0 \n\t"
1738 "psrad $15, %%mm2 \n\t"
1739 "packssdw %%mm2, %%mm0 \n\t"
1740 "packuswb %%mm0, %%mm0 \n\t"
1741 "movd %%mm0, (%1, %%"REG_a") \n\t"
1742 "add $4, %%"REG_a" \n\t"
1745 : "r" (dst+width), "g" ((x86_reg)-width)
1750 static inline void RENAME(bgr24ToUV_mmx)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src, long width, enum PixelFormat srcFormat)
1753 "movq 24(%4), %%mm6 \n\t"
1754 "mov %3, %%"REG_a" \n\t"
1755 "pxor %%mm7, %%mm7 \n\t"
1757 PREFETCH" 64(%0) \n\t"
1758 "movd (%0), %%mm0 \n\t"
1759 "movd 2(%0), %%mm1 \n\t"
1760 "punpcklbw %%mm7, %%mm0 \n\t"
1761 "punpcklbw %%mm7, %%mm1 \n\t"
1762 "movq %%mm0, %%mm2 \n\t"
1763 "movq %%mm1, %%mm3 \n\t"
1764 "pmaddwd (%4), %%mm0 \n\t"
1765 "pmaddwd 8(%4), %%mm1 \n\t"
1766 "pmaddwd 16(%4), %%mm2 \n\t"
1767 "pmaddwd %%mm6, %%mm3 \n\t"
1768 "paddd %%mm1, %%mm0 \n\t"
1769 "paddd %%mm3, %%mm2 \n\t"
1771 "movd 6(%0), %%mm1 \n\t"
1772 "movd 8(%0), %%mm3 \n\t"
1774 "punpcklbw %%mm7, %%mm1 \n\t"
1775 "punpcklbw %%mm7, %%mm3 \n\t"
1776 "movq %%mm1, %%mm4 \n\t"
1777 "movq %%mm3, %%mm5 \n\t"
1778 "pmaddwd (%4), %%mm1 \n\t"
1779 "pmaddwd 8(%4), %%mm3 \n\t"
1780 "pmaddwd 16(%4), %%mm4 \n\t"
1781 "pmaddwd %%mm6, %%mm5 \n\t"
1782 "paddd %%mm3, %%mm1 \n\t"
1783 "paddd %%mm5, %%mm4 \n\t"
1785 "movq "MANGLE(ff_bgr24toUVOffset)", %%mm3 \n\t"
1786 "paddd %%mm3, %%mm0 \n\t"
1787 "paddd %%mm3, %%mm2 \n\t"
1788 "paddd %%mm3, %%mm1 \n\t"
1789 "paddd %%mm3, %%mm4 \n\t"
1790 "psrad $15, %%mm0 \n\t"
1791 "psrad $15, %%mm2 \n\t"
1792 "psrad $15, %%mm1 \n\t"
1793 "psrad $15, %%mm4 \n\t"
1794 "packssdw %%mm1, %%mm0 \n\t"
1795 "packssdw %%mm4, %%mm2 \n\t"
1796 "packuswb %%mm0, %%mm0 \n\t"
1797 "packuswb %%mm2, %%mm2 \n\t"
1798 "movd %%mm0, (%1, %%"REG_a") \n\t"
1799 "movd %%mm2, (%2, %%"REG_a") \n\t"
1800 "add $4, %%"REG_a" \n\t"
1803 : "r" (dstU+width), "r" (dstV+width), "g" ((x86_reg)-width), "r"(ff_bgr24toUV[srcFormat == PIX_FMT_RGB24])
1808 static inline void RENAME(bgr24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1810 RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_BGR24);
1813 static inline void RENAME(bgr24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1815 RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_BGR24);
1816 assert(src1 == src2);
1819 static inline void RENAME(rgb24ToY)(uint8_t *dst, const uint8_t *src, long width, uint32_t *unused)
1821 RENAME(bgr24ToY_mmx)(dst, src, width, PIX_FMT_RGB24);
1824 static inline void RENAME(rgb24ToUV)(uint8_t *dstU, uint8_t *dstV, const uint8_t *src1, const uint8_t *src2, long width, uint32_t *unused)
1827 RENAME(bgr24ToUV_mmx)(dstU, dstV, src1, width, PIX_FMT_RGB24);
1831 // bilinear / bicubic scaling
1832 static inline void RENAME(hScale)(int16_t *dst, int dstW, const uint8_t *src, int srcW, int xInc,
1833 const int16_t *filter, const int16_t *filterPos, long filterSize)
1835 assert(filterSize % 4 == 0 && filterSize>0);
1836 if (filterSize==4) { // Always true for upscaling, sometimes for down, too.
1837 x86_reg counter= -2*dstW;
1839 filterPos-= counter/2;
1843 "push %%"REG_b" \n\t"
1845 "pxor %%mm7, %%mm7 \n\t"
1846 "push %%"REG_BP" \n\t" // we use 7 regs here ...
1847 "mov %%"REG_a", %%"REG_BP" \n\t"
1850 "movzwl (%2, %%"REG_BP"), %%eax \n\t"
1851 "movzwl 2(%2, %%"REG_BP"), %%ebx \n\t"
1852 "movq (%1, %%"REG_BP", 4), %%mm1 \n\t"
1853 "movq 8(%1, %%"REG_BP", 4), %%mm3 \n\t"
1854 "movd (%3, %%"REG_a"), %%mm0 \n\t"
1855 "movd (%3, %%"REG_b"), %%mm2 \n\t"
1856 "punpcklbw %%mm7, %%mm0 \n\t"
1857 "punpcklbw %%mm7, %%mm2 \n\t"
1858 "pmaddwd %%mm1, %%mm0 \n\t"
1859 "pmaddwd %%mm2, %%mm3 \n\t"
1860 "movq %%mm0, %%mm4 \n\t"
1861 "punpckldq %%mm3, %%mm0 \n\t"
1862 "punpckhdq %%mm3, %%mm4 \n\t"
1863 "paddd %%mm4, %%mm0 \n\t"
1864 "psrad $7, %%mm0 \n\t"
1865 "packssdw %%mm0, %%mm0 \n\t"
1866 "movd %%mm0, (%4, %%"REG_BP") \n\t"
1867 "add $4, %%"REG_BP" \n\t"
1870 "pop %%"REG_BP" \n\t"
1872 "pop %%"REG_b" \n\t"
1875 : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1880 } else if (filterSize==8) {
1881 x86_reg counter= -2*dstW;
1883 filterPos-= counter/2;
1887 "push %%"REG_b" \n\t"
1889 "pxor %%mm7, %%mm7 \n\t"
1890 "push %%"REG_BP" \n\t" // we use 7 regs here ...
1891 "mov %%"REG_a", %%"REG_BP" \n\t"
1894 "movzwl (%2, %%"REG_BP"), %%eax \n\t"
1895 "movzwl 2(%2, %%"REG_BP"), %%ebx \n\t"
1896 "movq (%1, %%"REG_BP", 8), %%mm1 \n\t"
1897 "movq 16(%1, %%"REG_BP", 8), %%mm3 \n\t"
1898 "movd (%3, %%"REG_a"), %%mm0 \n\t"
1899 "movd (%3, %%"REG_b"), %%mm2 \n\t"
1900 "punpcklbw %%mm7, %%mm0 \n\t"
1901 "punpcklbw %%mm7, %%mm2 \n\t"
1902 "pmaddwd %%mm1, %%mm0 \n\t"
1903 "pmaddwd %%mm2, %%mm3 \n\t"
1905 "movq 8(%1, %%"REG_BP", 8), %%mm1 \n\t"
1906 "movq 24(%1, %%"REG_BP", 8), %%mm5 \n\t"
1907 "movd 4(%3, %%"REG_a"), %%mm4 \n\t"
1908 "movd 4(%3, %%"REG_b"), %%mm2 \n\t"
1909 "punpcklbw %%mm7, %%mm4 \n\t"
1910 "punpcklbw %%mm7, %%mm2 \n\t"
1911 "pmaddwd %%mm1, %%mm4 \n\t"
1912 "pmaddwd %%mm2, %%mm5 \n\t"
1913 "paddd %%mm4, %%mm0 \n\t"
1914 "paddd %%mm5, %%mm3 \n\t"
1915 "movq %%mm0, %%mm4 \n\t"
1916 "punpckldq %%mm3, %%mm0 \n\t"
1917 "punpckhdq %%mm3, %%mm4 \n\t"
1918 "paddd %%mm4, %%mm0 \n\t"
1919 "psrad $7, %%mm0 \n\t"
1920 "packssdw %%mm0, %%mm0 \n\t"
1921 "movd %%mm0, (%4, %%"REG_BP") \n\t"
1922 "add $4, %%"REG_BP" \n\t"
1925 "pop %%"REG_BP" \n\t"
1927 "pop %%"REG_b" \n\t"
1930 : "c" (filter), "d" (filterPos), "S" (src), "D" (dst)
1936 const uint8_t *offset = src+filterSize;
1937 x86_reg counter= -2*dstW;
1938 //filter-= counter*filterSize/2;
1939 filterPos-= counter/2;
1942 "pxor %%mm7, %%mm7 \n\t"
1945 "mov %2, %%"REG_c" \n\t"
1946 "movzwl (%%"REG_c", %0), %%eax \n\t"
1947 "movzwl 2(%%"REG_c", %0), %%edx \n\t"
1948 "mov %5, %%"REG_c" \n\t"
1949 "pxor %%mm4, %%mm4 \n\t"
1950 "pxor %%mm5, %%mm5 \n\t"
1952 "movq (%1), %%mm1 \n\t"
1953 "movq (%1, %6), %%mm3 \n\t"
1954 "movd (%%"REG_c", %%"REG_a"), %%mm0 \n\t"
1955 "movd (%%"REG_c", %%"REG_d"), %%mm2 \n\t"
1956 "punpcklbw %%mm7, %%mm0 \n\t"
1957 "punpcklbw %%mm7, %%mm2 \n\t"
1958 "pmaddwd %%mm1, %%mm0 \n\t"
1959 "pmaddwd %%mm2, %%mm3 \n\t"
1960 "paddd %%mm3, %%mm5 \n\t"
1961 "paddd %%mm0, %%mm4 \n\t"
1963 "add $4, %%"REG_c" \n\t"
1964 "cmp %4, %%"REG_c" \n\t"
1967 "movq %%mm4, %%mm0 \n\t"
1968 "punpckldq %%mm5, %%mm4 \n\t"
1969 "punpckhdq %%mm5, %%mm0 \n\t"
1970 "paddd %%mm0, %%mm4 \n\t"
1971 "psrad $7, %%mm4 \n\t"
1972 "packssdw %%mm4, %%mm4 \n\t"
1973 "mov %3, %%"REG_a" \n\t"
1974 "movd %%mm4, (%%"REG_a", %0) \n\t"
1978 : "+r" (counter), "+r" (filter)
1979 : "m" (filterPos), "m" (dst), "m"(offset),
1980 "m" (src), "r" ((x86_reg)filterSize*2)
1981 : "%"REG_a, "%"REG_c, "%"REG_d
1986 #define FAST_BILINEAR_X86 \
1987 "subl %%edi, %%esi \n\t" /* src[xx+1] - src[xx] */ \
1988 "imull %%ecx, %%esi \n\t" /* (src[xx+1] - src[xx])*xalpha */ \
1989 "shll $16, %%edi \n\t" \
1990 "addl %%edi, %%esi \n\t" /* src[xx+1]*xalpha + src[xx]*(1-xalpha) */ \
1991 "mov %1, %%"REG_D"\n\t" \
1992 "shrl $9, %%esi \n\t" \
1994 static inline void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst,
1995 long dstWidth, const uint8_t *src, int srcW,
1998 #if COMPILE_TEMPLATE_MMX2
1999 int32_t *filterPos = c->hLumFilterPos;
2000 int16_t *filter = c->hLumFilter;
2001 int canMMX2BeUsed = c->canMMX2BeUsed;
2002 void *mmx2FilterCode= c->lumMmx2FilterCode;
2005 DECLARE_ALIGNED(8, uint64_t, ebxsave);
2007 if (canMMX2BeUsed) {
2010 "mov %%"REG_b", %5 \n\t"
2012 "pxor %%mm7, %%mm7 \n\t"
2013 "mov %0, %%"REG_c" \n\t"
2014 "mov %1, %%"REG_D" \n\t"
2015 "mov %2, %%"REG_d" \n\t"
2016 "mov %3, %%"REG_b" \n\t"
2017 "xor %%"REG_a", %%"REG_a" \n\t" // i
2018 PREFETCH" (%%"REG_c") \n\t"
2019 PREFETCH" 32(%%"REG_c") \n\t"
2020 PREFETCH" 64(%%"REG_c") \n\t"
2024 #define CALL_MMX2_FILTER_CODE \
2025 "movl (%%"REG_b"), %%esi \n\t"\
2027 "movl (%%"REG_b", %%"REG_a"), %%esi \n\t"\
2028 "add %%"REG_S", %%"REG_c" \n\t"\
2029 "add %%"REG_a", %%"REG_D" \n\t"\
2030 "xor %%"REG_a", %%"REG_a" \n\t"\
2034 #define CALL_MMX2_FILTER_CODE \
2035 "movl (%%"REG_b"), %%esi \n\t"\
2037 "addl (%%"REG_b", %%"REG_a"), %%"REG_c" \n\t"\
2038 "add %%"REG_a", %%"REG_D" \n\t"\
2039 "xor %%"REG_a", %%"REG_a" \n\t"\
2041 #endif /* ARCH_X86_64 */
2043 CALL_MMX2_FILTER_CODE
2044 CALL_MMX2_FILTER_CODE
2045 CALL_MMX2_FILTER_CODE
2046 CALL_MMX2_FILTER_CODE
2047 CALL_MMX2_FILTER_CODE
2048 CALL_MMX2_FILTER_CODE
2049 CALL_MMX2_FILTER_CODE
2050 CALL_MMX2_FILTER_CODE
2053 "mov %5, %%"REG_b" \n\t"
2055 :: "m" (src), "m" (dst), "m" (filter), "m" (filterPos),
2056 "m" (mmx2FilterCode)
2060 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2065 for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) dst[i] = src[srcW-1]*128;
2067 #endif /* COMPILE_TEMPLATE_MMX2 */
2068 x86_reg xInc_shr16 = xInc >> 16;
2069 uint16_t xInc_mask = xInc & 0xffff;
2070 x86_reg dstWidth_reg = dstWidth;
2071 //NO MMX just normal asm ...
2073 "xor %%"REG_a", %%"REG_a" \n\t" // i
2074 "xor %%"REG_d", %%"REG_d" \n\t" // xx
2075 "xorl %%ecx, %%ecx \n\t" // xalpha
2078 "movzbl (%0, %%"REG_d"), %%edi \n\t" //src[xx]
2079 "movzbl 1(%0, %%"REG_d"), %%esi \n\t" //src[xx+1]
2081 "movw %%si, (%%"REG_D", %%"REG_a", 2) \n\t"
2082 "addw %4, %%cx \n\t" //xalpha += xInc&0xFFFF
2083 "adc %3, %%"REG_d" \n\t" //xx+= xInc>>16 + carry
2085 "movzbl (%0, %%"REG_d"), %%edi \n\t" //src[xx]
2086 "movzbl 1(%0, %%"REG_d"), %%esi \n\t" //src[xx+1]
2088 "movw %%si, 2(%%"REG_D", %%"REG_a", 2) \n\t"
2089 "addw %4, %%cx \n\t" //xalpha += xInc&0xFFFF
2090 "adc %3, %%"REG_d" \n\t" //xx+= xInc>>16 + carry
2093 "add $2, %%"REG_a" \n\t"
2094 "cmp %2, %%"REG_a" \n\t"
2098 :: "r" (src), "m" (dst), "m" (dstWidth_reg), "m" (xInc_shr16), "m" (xInc_mask)
2099 : "%"REG_a, "%"REG_d, "%ecx", "%"REG_D, "%esi"
2101 #if COMPILE_TEMPLATE_MMX2
2102 } //if MMX2 can't be used
2106 static inline void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst,
2107 long dstWidth, const uint8_t *src1,
2108 const uint8_t *src2, int srcW, int xInc)
2110 #if COMPILE_TEMPLATE_MMX2
2111 int32_t *filterPos = c->hChrFilterPos;
2112 int16_t *filter = c->hChrFilter;
2113 int canMMX2BeUsed = c->canMMX2BeUsed;
2114 void *mmx2FilterCode= c->chrMmx2FilterCode;
2117 DECLARE_ALIGNED(8, uint64_t, ebxsave);
2119 if (canMMX2BeUsed) {
2122 "mov %%"REG_b", %6 \n\t"
2124 "pxor %%mm7, %%mm7 \n\t"
2125 "mov %0, %%"REG_c" \n\t"
2126 "mov %1, %%"REG_D" \n\t"
2127 "mov %2, %%"REG_d" \n\t"
2128 "mov %3, %%"REG_b" \n\t"
2129 "xor %%"REG_a", %%"REG_a" \n\t" // i
2130 PREFETCH" (%%"REG_c") \n\t"
2131 PREFETCH" 32(%%"REG_c") \n\t"
2132 PREFETCH" 64(%%"REG_c") \n\t"
2134 CALL_MMX2_FILTER_CODE
2135 CALL_MMX2_FILTER_CODE
2136 CALL_MMX2_FILTER_CODE
2137 CALL_MMX2_FILTER_CODE
2138 "xor %%"REG_a", %%"REG_a" \n\t" // i
2139 "mov %5, %%"REG_c" \n\t" // src
2140 "mov %1, %%"REG_D" \n\t" // buf1
2141 "add $"AV_STRINGIFY(VOF)", %%"REG_D" \n\t"
2142 PREFETCH" (%%"REG_c") \n\t"
2143 PREFETCH" 32(%%"REG_c") \n\t"
2144 PREFETCH" 64(%%"REG_c") \n\t"
2146 CALL_MMX2_FILTER_CODE
2147 CALL_MMX2_FILTER_CODE
2148 CALL_MMX2_FILTER_CODE
2149 CALL_MMX2_FILTER_CODE
2152 "mov %6, %%"REG_b" \n\t"
2154 :: "m" (src1), "m" (dst), "m" (filter), "m" (filterPos),
2155 "m" (mmx2FilterCode), "m" (src2)
2159 : "%"REG_a, "%"REG_c, "%"REG_d, "%"REG_S, "%"REG_D
2164 for (i=dstWidth-1; (i*xInc)>>16 >=srcW-1; i--) {
2165 //printf("%d %d %d\n", dstWidth, i, srcW);
2166 dst[i] = src1[srcW-1]*128;
2167 dst[i+VOFW] = src2[srcW-1]*128;
2170 #endif /* COMPILE_TEMPLATE_MMX2 */
2171 x86_reg xInc_shr16 = (x86_reg) (xInc >> 16);
2172 uint16_t xInc_mask = xInc & 0xffff;
2173 x86_reg dstWidth_reg = dstWidth;
2175 "xor %%"REG_a", %%"REG_a" \n\t" // i
2176 "xor %%"REG_d", %%"REG_d" \n\t" // xx
2177 "xorl %%ecx, %%ecx \n\t" // xalpha
2180 "mov %0, %%"REG_S" \n\t"
2181 "movzbl (%%"REG_S", %%"REG_d"), %%edi \n\t" //src[xx]
2182 "movzbl 1(%%"REG_S", %%"REG_d"), %%esi \n\t" //src[xx+1]
2184 "movw %%si, (%%"REG_D", %%"REG_a", 2) \n\t"
2186 "movzbl (%5, %%"REG_d"), %%edi \n\t" //src[xx]
2187 "movzbl 1(%5, %%"REG_d"), %%esi \n\t" //src[xx+1]
2189 "movw %%si, "AV_STRINGIFY(VOF)"(%%"REG_D", %%"REG_a", 2) \n\t"
2191 "addw %4, %%cx \n\t" //xalpha += xInc&0xFFFF
2192 "adc %3, %%"REG_d" \n\t" //xx+= xInc>>16 + carry
2193 "add $1, %%"REG_a" \n\t"
2194 "cmp %2, %%"REG_a" \n\t"
2197 /* GCC 3.3 makes MPlayer crash on IA-32 machines when using "g" operand here,
2198 which is needed to support GCC 4.0. */
2199 #if ARCH_X86_64 && AV_GCC_VERSION_AT_LEAST(3,4)
2200 :: "m" (src1), "m" (dst), "g" (dstWidth_reg), "m" (xInc_shr16), "m" (xInc_mask),
2202 :: "m" (src1), "m" (dst), "m" (dstWidth_reg), "m" (xInc_shr16), "m" (xInc_mask),
2205 : "%"REG_a, "%"REG_d, "%ecx", "%"REG_D, "%esi"
2207 #if COMPILE_TEMPLATE_MMX2
2208 } //if MMX2 can't be used
2212 #define DEBUG_SWSCALE_BUFFERS 0
2213 #define DEBUG_BUFFERS(...) if (DEBUG_SWSCALE_BUFFERS) av_log(c, AV_LOG_DEBUG, __VA_ARGS__)
2215 static int RENAME(swScale)(SwsContext *c, const uint8_t* src[], int srcStride[], int srcSliceY,
2216 int srcSliceH, uint8_t* dst[], int dstStride[])
2218 /* load a few things into local vars to make the code more readable? and faster */
2219 const int srcW= c->srcW;
2220 const int dstW= c->dstW;
2221 const int dstH= c->dstH;
2222 const int chrDstW= c->chrDstW;
2223 const int chrSrcW= c->chrSrcW;
2224 const int lumXInc= c->lumXInc;
2225 const int chrXInc= c->chrXInc;
2226 const enum PixelFormat dstFormat= c->dstFormat;
2227 const int flags= c->flags;
2228 int16_t *vLumFilterPos= c->vLumFilterPos;
2229 int16_t *vChrFilterPos= c->vChrFilterPos;
2230 int16_t *hLumFilterPos= c->hLumFilterPos;
2231 int16_t *hChrFilterPos= c->hChrFilterPos;
2232 int16_t *vLumFilter= c->vLumFilter;
2233 int16_t *vChrFilter= c->vChrFilter;
2234 int16_t *hLumFilter= c->hLumFilter;
2235 int16_t *hChrFilter= c->hChrFilter;
2236 int32_t *lumMmxFilter= c->lumMmxFilter;
2237 int32_t *chrMmxFilter= c->chrMmxFilter;
2238 int32_t av_unused *alpMmxFilter= c->alpMmxFilter;
2239 const int vLumFilterSize= c->vLumFilterSize;
2240 const int vChrFilterSize= c->vChrFilterSize;
2241 const int hLumFilterSize= c->hLumFilterSize;
2242 const int hChrFilterSize= c->hChrFilterSize;
2243 int16_t **lumPixBuf= c->lumPixBuf;
2244 int16_t **chrPixBuf= c->chrPixBuf;
2245 int16_t **alpPixBuf= c->alpPixBuf;
2246 const int vLumBufSize= c->vLumBufSize;
2247 const int vChrBufSize= c->vChrBufSize;
2248 uint8_t *formatConvBuffer= c->formatConvBuffer;
2249 const int chrSrcSliceY= srcSliceY >> c->chrSrcVSubSample;
2250 const int chrSrcSliceH= -((-srcSliceH) >> c->chrSrcVSubSample);
2252 uint32_t *pal=c->pal_yuv;
2254 /* vars which will change and which we need to store back in the context */
2256 int lumBufIndex= c->lumBufIndex;
2257 int chrBufIndex= c->chrBufIndex;
2258 int lastInLumBuf= c->lastInLumBuf;
2259 int lastInChrBuf= c->lastInChrBuf;
2261 if (isPacked(c->srcFormat)) {
2269 srcStride[3]= srcStride[0];
2271 srcStride[1]<<= c->vChrDrop;
2272 srcStride[2]<<= c->vChrDrop;
2274 DEBUG_BUFFERS("swScale() %p[%d] %p[%d] %p[%d] %p[%d] -> %p[%d] %p[%d] %p[%d] %p[%d]\n",
2275 src[0], srcStride[0], src[1], srcStride[1], src[2], srcStride[2], src[3], srcStride[3],
2276 dst[0], dstStride[0], dst[1], dstStride[1], dst[2], dstStride[2], dst[3], dstStride[3]);
2277 DEBUG_BUFFERS("srcSliceY: %d srcSliceH: %d dstY: %d dstH: %d\n",
2278 srcSliceY, srcSliceH, dstY, dstH);
2279 DEBUG_BUFFERS("vLumFilterSize: %d vLumBufSize: %d vChrFilterSize: %d vChrBufSize: %d\n",
2280 vLumFilterSize, vLumBufSize, vChrFilterSize, vChrBufSize);
2282 if (dstStride[0]%8 !=0 || dstStride[1]%8 !=0 || dstStride[2]%8 !=0 || dstStride[3]%8 != 0) {
2283 static int warnedAlready=0; //FIXME move this into the context perhaps
2284 if (flags & SWS_PRINT_INFO && !warnedAlready) {
2285 av_log(c, AV_LOG_WARNING, "Warning: dstStride is not aligned!\n"
2286 " ->cannot do aligned memory accesses anymore\n");
2291 /* Note the user might start scaling the picture in the middle so this
2292 will not get executed. This is not really intended but works
2293 currently, so people might do it. */
2294 if (srcSliceY ==0) {
2304 for (;dstY < dstH; dstY++) {
2305 unsigned char *dest =dst[0]+dstStride[0]*dstY;
2306 const int chrDstY= dstY>>c->chrDstVSubSample;
2307 unsigned char *uDest=dst[1]+dstStride[1]*chrDstY;
2308 unsigned char *vDest=dst[2]+dstStride[2]*chrDstY;
2309 unsigned char *aDest=(CONFIG_SWSCALE_ALPHA && alpPixBuf) ? dst[3]+dstStride[3]*dstY : NULL;
2311 const int firstLumSrcY= vLumFilterPos[dstY]; //First line needed as input
2312 const int firstLumSrcY2= vLumFilterPos[FFMIN(dstY | ((1<<c->chrDstVSubSample) - 1), dstH-1)];
2313 const int firstChrSrcY= vChrFilterPos[chrDstY]; //First line needed as input
2314 int lastLumSrcY= firstLumSrcY + vLumFilterSize -1; // Last line needed as input
2315 int lastLumSrcY2=firstLumSrcY2+ vLumFilterSize -1; // Last line needed as input
2316 int lastChrSrcY= firstChrSrcY + vChrFilterSize -1; // Last line needed as input
2319 //handle holes (FAST_BILINEAR & weird filters)
2320 if (firstLumSrcY > lastInLumBuf) lastInLumBuf= firstLumSrcY-1;
2321 if (firstChrSrcY > lastInChrBuf) lastInChrBuf= firstChrSrcY-1;
2322 assert(firstLumSrcY >= lastInLumBuf - vLumBufSize + 1);
2323 assert(firstChrSrcY >= lastInChrBuf - vChrBufSize + 1);
2325 DEBUG_BUFFERS("dstY: %d\n", dstY);
2326 DEBUG_BUFFERS("\tfirstLumSrcY: %d lastLumSrcY: %d lastInLumBuf: %d\n",
2327 firstLumSrcY, lastLumSrcY, lastInLumBuf);
2328 DEBUG_BUFFERS("\tfirstChrSrcY: %d lastChrSrcY: %d lastInChrBuf: %d\n",
2329 firstChrSrcY, lastChrSrcY, lastInChrBuf);
2331 // Do we have enough lines in this slice to output the dstY line
2332 enough_lines = lastLumSrcY2 < srcSliceY + srcSliceH && lastChrSrcY < -((-srcSliceY - srcSliceH)>>c->chrSrcVSubSample);
2334 if (!enough_lines) {
2335 lastLumSrcY = srcSliceY + srcSliceH - 1;
2336 lastChrSrcY = chrSrcSliceY + chrSrcSliceH - 1;
2337 DEBUG_BUFFERS("buffering slice: lastLumSrcY %d lastChrSrcY %d\n",
2338 lastLumSrcY, lastChrSrcY);
2341 //Do horizontal scaling
2342 while(lastInLumBuf < lastLumSrcY) {
2343 const uint8_t *src1= src[0]+(lastInLumBuf + 1 - srcSliceY)*srcStride[0];
2344 const uint8_t *src2= src[3]+(lastInLumBuf + 1 - srcSliceY)*srcStride[3];
2346 assert(lumBufIndex < 2*vLumBufSize);
2347 assert(lastInLumBuf + 1 - srcSliceY < srcSliceH);
2348 assert(lastInLumBuf + 1 - srcSliceY >= 0);
2349 hyscale_c(c, lumPixBuf[ lumBufIndex ], dstW, src1, srcW, lumXInc,
2350 hLumFilter, hLumFilterPos, hLumFilterSize,
2353 if (CONFIG_SWSCALE_ALPHA && alpPixBuf)
2354 hyscale_c(c, alpPixBuf[ lumBufIndex ], dstW, src2, srcW, lumXInc,
2355 hLumFilter, hLumFilterPos, hLumFilterSize,
2359 DEBUG_BUFFERS("\t\tlumBufIndex %d: lastInLumBuf: %d\n",
2360 lumBufIndex, lastInLumBuf);
2362 while(lastInChrBuf < lastChrSrcY) {
2363 const uint8_t *src1= src[1]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[1];
2364 const uint8_t *src2= src[2]+(lastInChrBuf + 1 - chrSrcSliceY)*srcStride[2];
2366 assert(chrBufIndex < 2*vChrBufSize);
2367 assert(lastInChrBuf + 1 - chrSrcSliceY < (chrSrcSliceH));
2368 assert(lastInChrBuf + 1 - chrSrcSliceY >= 0);
2369 //FIXME replace parameters through context struct (some at least)
2371 if (c->needs_hcscale)
2372 hcscale_c(c, chrPixBuf[ chrBufIndex ], chrDstW, src1, src2, chrSrcW, chrXInc,
2373 hChrFilter, hChrFilterPos, hChrFilterSize,
2377 DEBUG_BUFFERS("\t\tchrBufIndex %d: lastInChrBuf: %d\n",
2378 chrBufIndex, lastInChrBuf);
2380 //wrap buf index around to stay inside the ring buffer
2381 if (lumBufIndex >= vLumBufSize) lumBufIndex-= vLumBufSize;
2382 if (chrBufIndex >= vChrBufSize) chrBufIndex-= vChrBufSize;
2384 break; //we can't output a dstY line so let's try with the next slice
2386 c->blueDither= ff_dither8[dstY&1];
2387 if (c->dstFormat == PIX_FMT_RGB555 || c->dstFormat == PIX_FMT_BGR555)
2388 c->greenDither= ff_dither8[dstY&1];
2390 c->greenDither= ff_dither4[dstY&1];
2391 c->redDither= ff_dither8[(dstY+1)&1];
2392 if (dstY < dstH-2) {
2393 const int16_t **lumSrcPtr= (const int16_t **) lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2394 const int16_t **chrSrcPtr= (const int16_t **) chrPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2395 const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;
2397 if (flags & SWS_ACCURATE_RND) {
2398 int s= APCK_SIZE / 8;
2399 for (i=0; i<vLumFilterSize; i+=2) {
2400 *(const void**)&lumMmxFilter[s*i ]= lumSrcPtr[i ];
2401 *(const void**)&lumMmxFilter[s*i+APCK_PTR2/4 ]= lumSrcPtr[i+(vLumFilterSize>1)];
2402 lumMmxFilter[s*i+APCK_COEF/4 ]=
2403 lumMmxFilter[s*i+APCK_COEF/4+1]= vLumFilter[dstY*vLumFilterSize + i ]
2404 + (vLumFilterSize>1 ? vLumFilter[dstY*vLumFilterSize + i + 1]<<16 : 0);
2405 if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2406 *(const void**)&alpMmxFilter[s*i ]= alpSrcPtr[i ];
2407 *(const void**)&alpMmxFilter[s*i+APCK_PTR2/4 ]= alpSrcPtr[i+(vLumFilterSize>1)];
2408 alpMmxFilter[s*i+APCK_COEF/4 ]=
2409 alpMmxFilter[s*i+APCK_COEF/4+1]= lumMmxFilter[s*i+APCK_COEF/4 ];
2412 for (i=0; i<vChrFilterSize; i+=2) {
2413 *(const void**)&chrMmxFilter[s*i ]= chrSrcPtr[i ];
2414 *(const void**)&chrMmxFilter[s*i+APCK_PTR2/4 ]= chrSrcPtr[i+(vChrFilterSize>1)];
2415 chrMmxFilter[s*i+APCK_COEF/4 ]=
2416 chrMmxFilter[s*i+APCK_COEF/4+1]= vChrFilter[chrDstY*vChrFilterSize + i ]
2417 + (vChrFilterSize>1 ? vChrFilter[chrDstY*vChrFilterSize + i + 1]<<16 : 0);
2420 for (i=0; i<vLumFilterSize; i++) {
2421 lumMmxFilter[4*i+0]= (int32_t)lumSrcPtr[i];
2422 lumMmxFilter[4*i+1]= (uint64_t)lumSrcPtr[i] >> 32;
2423 lumMmxFilter[4*i+2]=
2424 lumMmxFilter[4*i+3]=
2425 ((uint16_t)vLumFilter[dstY*vLumFilterSize + i])*0x10001;
2426 if (CONFIG_SWSCALE_ALPHA && alpPixBuf) {
2427 alpMmxFilter[4*i+0]= (int32_t)alpSrcPtr[i];
2428 alpMmxFilter[4*i+1]= (uint64_t)alpSrcPtr[i] >> 32;
2429 alpMmxFilter[4*i+2]=
2430 alpMmxFilter[4*i+3]= lumMmxFilter[4*i+2];
2433 for (i=0; i<vChrFilterSize; i++) {
2434 chrMmxFilter[4*i+0]= (int32_t)chrSrcPtr[i];
2435 chrMmxFilter[4*i+1]= (uint64_t)chrSrcPtr[i] >> 32;
2436 chrMmxFilter[4*i+2]=
2437 chrMmxFilter[4*i+3]=
2438 ((uint16_t)vChrFilter[chrDstY*vChrFilterSize + i])*0x10001;
2441 if (dstFormat == PIX_FMT_NV12 || dstFormat == PIX_FMT_NV21) {
2442 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2443 if (dstY&chrSkipMask) uDest= NULL; //FIXME split functions in lumi / chromi
2445 vLumFilter+dstY*vLumFilterSize , lumSrcPtr, vLumFilterSize,
2446 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2447 dest, uDest, dstW, chrDstW, dstFormat);
2448 } else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12 like
2449 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2450 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
2451 if (is16BPS(dstFormat) || is9_OR_10BPS(dstFormat)) {
2453 vLumFilter+dstY*vLumFilterSize , lumSrcPtr, vLumFilterSize,
2454 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2455 alpSrcPtr, (uint16_t *) dest, (uint16_t *) uDest, (uint16_t *) vDest, (uint16_t *) aDest, dstW, chrDstW,
2457 } else if (vLumFilterSize == 1 && vChrFilterSize == 1) { // unscaled YV12
2458 const int16_t *lumBuf = lumSrcPtr[0];
2459 const int16_t *chrBuf= chrSrcPtr[0];
2460 const int16_t *alpBuf= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? alpSrcPtr[0] : NULL;
2461 c->yuv2yuv1(c, lumBuf, chrBuf, alpBuf, dest, uDest, vDest, aDest, dstW, chrDstW);
2462 } else { //General YV12
2464 vLumFilter+dstY*vLumFilterSize , lumSrcPtr, vLumFilterSize,
2465 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2466 alpSrcPtr, dest, uDest, vDest, aDest, dstW, chrDstW);
2469 assert(lumSrcPtr + vLumFilterSize - 1 < lumPixBuf + vLumBufSize*2);
2470 assert(chrSrcPtr + vChrFilterSize - 1 < chrPixBuf + vChrBufSize*2);
2471 if (vLumFilterSize == 1 && vChrFilterSize == 2) { //unscaled RGB
2472 int chrAlpha= vChrFilter[2*dstY+1];
2473 if(flags & SWS_FULL_CHR_H_INT) {
2474 yuv2rgbXinC_full(c, //FIXME write a packed1_full function
2475 vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2476 vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2477 alpSrcPtr, dest, dstW, dstY);
2479 c->yuv2packed1(c, *lumSrcPtr, *chrSrcPtr, *(chrSrcPtr+1),
2480 alpPixBuf ? *alpSrcPtr : NULL,
2481 dest, dstW, chrAlpha, dstFormat, flags, dstY);
2483 } else if (vLumFilterSize == 2 && vChrFilterSize == 2) { //bilinear upscale RGB
2484 int lumAlpha= vLumFilter[2*dstY+1];
2485 int chrAlpha= vChrFilter[2*dstY+1];
2487 lumMmxFilter[3]= vLumFilter[2*dstY ]*0x10001;
2489 chrMmxFilter[3]= vChrFilter[2*chrDstY]*0x10001;
2490 if(flags & SWS_FULL_CHR_H_INT) {
2491 yuv2rgbXinC_full(c, //FIXME write a packed2_full function
2492 vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2493 vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2494 alpSrcPtr, dest, dstW, dstY);
2496 c->yuv2packed2(c, *lumSrcPtr, *(lumSrcPtr+1), *chrSrcPtr, *(chrSrcPtr+1),
2497 alpPixBuf ? *alpSrcPtr : NULL, alpPixBuf ? *(alpSrcPtr+1) : NULL,
2498 dest, dstW, lumAlpha, chrAlpha, dstY);
2500 } else { //general RGB
2501 if(flags & SWS_FULL_CHR_H_INT) {
2503 vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2504 vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2505 alpSrcPtr, dest, dstW, dstY);
2508 vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2509 vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2510 alpSrcPtr, dest, dstW, dstY);
2514 } else { // hmm looks like we can't use MMX here without overwriting this array's tail
2515 const int16_t **lumSrcPtr= (const int16_t **)lumPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize;
2516 const int16_t **chrSrcPtr= (const int16_t **)chrPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize;
2517 const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **)alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL;
2518 if (dstFormat == PIX_FMT_NV12 || dstFormat == PIX_FMT_NV21) {
2519 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2520 if (dstY&chrSkipMask) uDest= NULL; //FIXME split functions in lumi / chromi
2522 vLumFilter+dstY*vLumFilterSize , lumSrcPtr, vLumFilterSize,
2523 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2524 dest, uDest, dstW, chrDstW, dstFormat);
2525 } else if (isPlanarYUV(dstFormat) || dstFormat==PIX_FMT_GRAY8) { //YV12
2526 const int chrSkipMask= (1<<c->chrDstVSubSample)-1;
2527 if ((dstY&chrSkipMask) || isGray(dstFormat)) uDest=vDest= NULL; //FIXME split functions in lumi / chromi
2528 if (is16BPS(dstFormat) || is9_OR_10BPS(dstFormat)) {
2530 vLumFilter+dstY*vLumFilterSize , lumSrcPtr, vLumFilterSize,
2531 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2532 alpSrcPtr, (uint16_t *) dest, (uint16_t *) uDest, (uint16_t *) vDest, (uint16_t *) aDest, dstW, chrDstW,
2536 vLumFilter+dstY*vLumFilterSize , lumSrcPtr, vLumFilterSize,
2537 vChrFilter+chrDstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2538 alpSrcPtr, dest, uDest, vDest, aDest, dstW, chrDstW);
2541 assert(lumSrcPtr + vLumFilterSize - 1 < lumPixBuf + vLumBufSize*2);
2542 assert(chrSrcPtr + vChrFilterSize - 1 < chrPixBuf + vChrBufSize*2);
2543 if(flags & SWS_FULL_CHR_H_INT) {
2545 vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2546 vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2547 alpSrcPtr, dest, dstW, dstY);
2550 vLumFilter+dstY*vLumFilterSize, lumSrcPtr, vLumFilterSize,
2551 vChrFilter+dstY*vChrFilterSize, chrSrcPtr, vChrFilterSize,
2552 alpSrcPtr, dest, dstW, dstY);
2558 if ((dstFormat == PIX_FMT_YUVA420P) && !alpPixBuf)
2559 fillPlane(dst[3], dstStride[3], dstW, dstY-lastDstY, lastDstY, 255);
2561 if (COMPILE_TEMPLATE_MMX2) __asm__ volatile("sfence":::"memory");
2562 /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */
2563 if (COMPILE_TEMPLATE_AMD3DNOW) __asm__ volatile("femms" :::"memory");
2564 else __asm__ volatile("emms" :::"memory");
2566 /* store changed local vars back in the context */
2568 c->lumBufIndex= lumBufIndex;
2569 c->chrBufIndex= chrBufIndex;
2570 c->lastInLumBuf= lastInLumBuf;
2571 c->lastInChrBuf= lastInChrBuf;
2573 return dstY - lastDstY;
2576 static void RENAME(sws_init_swScale)(SwsContext *c)
2578 enum PixelFormat srcFormat = c->srcFormat;
2580 c->yuv2yuv1 = RENAME(yuv2yuv1 );
2581 c->yuv2yuvX = RENAME(yuv2yuvX );
2582 c->yuv2packed1 = RENAME(yuv2packed1 );
2583 c->yuv2packed2 = RENAME(yuv2packed2 );
2584 c->yuv2packedX = RENAME(yuv2packedX );
2586 c->hScale = RENAME(hScale );
2588 // Use the new MMX scaler if the MMX2 one can't be used (it is faster than the x86 ASM one).
2589 if (c->flags & SWS_FAST_BILINEAR && c->canMMX2BeUsed)
2591 c->hyscale_fast = RENAME(hyscale_fast);
2592 c->hcscale_fast = RENAME(hcscale_fast);
2594 c->hyscale_fast = NULL;
2595 c->hcscale_fast = NULL;
2599 case PIX_FMT_YUYV422 : c->chrToYV12 = RENAME(yuy2ToUV); break;
2600 case PIX_FMT_UYVY422 : c->chrToYV12 = RENAME(uyvyToUV); break;
2601 case PIX_FMT_NV12 : c->chrToYV12 = RENAME(nv12ToUV); break;
2602 case PIX_FMT_NV21 : c->chrToYV12 = RENAME(nv21ToUV); break;
2603 case PIX_FMT_YUV420P16BE:
2604 case PIX_FMT_YUV422P16BE:
2605 case PIX_FMT_YUV444P16BE: c->chrToYV12 = RENAME(BEToUV); break;
2606 case PIX_FMT_YUV420P16LE:
2607 case PIX_FMT_YUV422P16LE:
2608 case PIX_FMT_YUV444P16LE: c->chrToYV12 = RENAME(LEToUV); break;
2611 if (!c->chrSrcHSubSample) {
2613 case PIX_FMT_BGR24 : c->chrToYV12 = RENAME(bgr24ToUV); break;
2614 case PIX_FMT_RGB24 : c->chrToYV12 = RENAME(rgb24ToUV); break;
2619 switch (srcFormat) {
2620 case PIX_FMT_YUYV422 :
2621 case PIX_FMT_YUV420P16BE:
2622 case PIX_FMT_YUV422P16BE:
2623 case PIX_FMT_YUV444P16BE:
2624 case PIX_FMT_Y400A :
2625 case PIX_FMT_GRAY16BE : c->lumToYV12 = RENAME(yuy2ToY); break;
2626 case PIX_FMT_UYVY422 :
2627 case PIX_FMT_YUV420P16LE:
2628 case PIX_FMT_YUV422P16LE:
2629 case PIX_FMT_YUV444P16LE:
2630 case PIX_FMT_GRAY16LE : c->lumToYV12 = RENAME(uyvyToY); break;
2631 case PIX_FMT_BGR24 : c->lumToYV12 = RENAME(bgr24ToY); break;
2632 case PIX_FMT_RGB24 : c->lumToYV12 = RENAME(rgb24ToY); break;
2636 switch (srcFormat) {
2637 case PIX_FMT_Y400A : c->alpToYV12 = RENAME(yuy2ToY); break;