vda: Check the correct pointer for buffer allocation
[ffmpeg.git] / libavcodec / h264dsp.c
1 /*
2  * H.26L/H.264/AVC/JVT/14496-10/... encoder/decoder
3  * Copyright (c) 2003-2010 Michael Niedermayer <michaelni@gmx.at>
4  *
5  * This file is part of Libav.
6  *
7  * Libav is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * Libav is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with Libav; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21
22 /**
23  * @file
24  * H.264 / AVC / MPEG4 part10 DSP functions.
25  * @author Michael Niedermayer <michaelni@gmx.at>
26  */
27
28 #include <stdint.h>
29
30 #include "libavutil/attributes.h"
31 #include "avcodec.h"
32 #include "h264dsp.h"
33 #include "h264idct.h"
34 #include "startcode.h"
35 #include "libavutil/common.h"
36
37 #define BIT_DEPTH 8
38 #include "h264dsp_template.c"
39 #undef BIT_DEPTH
40
41 #define BIT_DEPTH 9
42 #include "h264dsp_template.c"
43 #undef BIT_DEPTH
44
45 #define BIT_DEPTH 10
46 #include "h264dsp_template.c"
47 #undef BIT_DEPTH
48
49 #define BIT_DEPTH 8
50 #include "h264addpx_template.c"
51 #undef BIT_DEPTH
52
53 #define BIT_DEPTH 16
54 #include "h264addpx_template.c"
55 #undef BIT_DEPTH
56
57 av_cold void ff_h264dsp_init(H264DSPContext *c, const int bit_depth,
58                              const int chroma_format_idc)
59 {
60 #undef FUNC
61 #define FUNC(a, depth) a ## _ ## depth ## _c
62
63 #define ADDPX_DSP(depth) \
64     c->h264_add_pixels4_clear = FUNC(ff_h264_add_pixels4, depth);\
65     c->h264_add_pixels8_clear = FUNC(ff_h264_add_pixels8, depth)
66
67     if (bit_depth > 8 && bit_depth <= 16) {
68         ADDPX_DSP(16);
69     } else {
70         ADDPX_DSP(8);
71     }
72
73 #define H264_DSP(depth) \
74     c->h264_idct_add= FUNC(ff_h264_idct_add, depth);\
75     c->h264_idct8_add= FUNC(ff_h264_idct8_add, depth);\
76     c->h264_idct_dc_add= FUNC(ff_h264_idct_dc_add, depth);\
77     c->h264_idct8_dc_add= FUNC(ff_h264_idct8_dc_add, depth);\
78     c->h264_idct_add16     = FUNC(ff_h264_idct_add16, depth);\
79     c->h264_idct8_add4     = FUNC(ff_h264_idct8_add4, depth);\
80     if (chroma_format_idc <= 1)\
81         c->h264_idct_add8  = FUNC(ff_h264_idct_add8, depth);\
82     else\
83         c->h264_idct_add8  = FUNC(ff_h264_idct_add8_422, depth);\
84     c->h264_idct_add16intra= FUNC(ff_h264_idct_add16intra, depth);\
85     c->h264_luma_dc_dequant_idct= FUNC(ff_h264_luma_dc_dequant_idct, depth);\
86     if (chroma_format_idc <= 1)\
87         c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma_dc_dequant_idct, depth);\
88     else\
89         c->h264_chroma_dc_dequant_idct= FUNC(ff_h264_chroma422_dc_dequant_idct, depth);\
90 \
91     c->weight_h264_pixels_tab[0]= FUNC(weight_h264_pixels16, depth);\
92     c->weight_h264_pixels_tab[1]= FUNC(weight_h264_pixels8, depth);\
93     c->weight_h264_pixels_tab[2]= FUNC(weight_h264_pixels4, depth);\
94     c->weight_h264_pixels_tab[3]= FUNC(weight_h264_pixels2, depth);\
95     c->biweight_h264_pixels_tab[0]= FUNC(biweight_h264_pixels16, depth);\
96     c->biweight_h264_pixels_tab[1]= FUNC(biweight_h264_pixels8, depth);\
97     c->biweight_h264_pixels_tab[2]= FUNC(biweight_h264_pixels4, depth);\
98     c->biweight_h264_pixels_tab[3]= FUNC(biweight_h264_pixels2, depth);\
99 \
100     c->h264_v_loop_filter_luma= FUNC(h264_v_loop_filter_luma, depth);\
101     c->h264_h_loop_filter_luma= FUNC(h264_h_loop_filter_luma, depth);\
102     c->h264_h_loop_filter_luma_mbaff= FUNC(h264_h_loop_filter_luma_mbaff, depth);\
103     c->h264_v_loop_filter_luma_intra= FUNC(h264_v_loop_filter_luma_intra, depth);\
104     c->h264_h_loop_filter_luma_intra= FUNC(h264_h_loop_filter_luma_intra, depth);\
105     c->h264_h_loop_filter_luma_mbaff_intra= FUNC(h264_h_loop_filter_luma_mbaff_intra, depth);\
106     c->h264_v_loop_filter_chroma= FUNC(h264_v_loop_filter_chroma, depth);\
107     if (chroma_format_idc <= 1)\
108         c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma, depth);\
109     else\
110         c->h264_h_loop_filter_chroma= FUNC(h264_h_loop_filter_chroma422, depth);\
111     if (chroma_format_idc <= 1)\
112         c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma_mbaff, depth);\
113     else\
114         c->h264_h_loop_filter_chroma_mbaff= FUNC(h264_h_loop_filter_chroma422_mbaff, depth);\
115     c->h264_v_loop_filter_chroma_intra= FUNC(h264_v_loop_filter_chroma_intra, depth);\
116     if (chroma_format_idc <= 1)\
117         c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma_intra, depth);\
118     else\
119         c->h264_h_loop_filter_chroma_intra= FUNC(h264_h_loop_filter_chroma422_intra, depth);\
120     if (chroma_format_idc <= 1)\
121         c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma_mbaff_intra, depth);\
122     else\
123         c->h264_h_loop_filter_chroma_mbaff_intra= FUNC(h264_h_loop_filter_chroma422_mbaff_intra, depth);\
124     c->h264_loop_filter_strength= NULL;
125
126     switch (bit_depth) {
127     case 9:
128         H264_DSP(9);
129         break;
130     case 10:
131         H264_DSP(10);
132         break;
133     default:
134         H264_DSP(8);
135         break;
136     }
137     c->startcode_find_candidate = ff_startcode_find_candidate_c;
138
139     if (ARCH_AARCH64) ff_h264dsp_init_aarch64(c, bit_depth, chroma_format_idc);
140     if (ARCH_ARM) ff_h264dsp_init_arm(c, bit_depth, chroma_format_idc);
141     if (ARCH_PPC) ff_h264dsp_init_ppc(c, bit_depth, chroma_format_idc);
142     if (ARCH_X86) ff_h264dsp_init_x86(c, bit_depth, chroma_format_idc);
143 }