stereo3d: Support view type for frame sequence type
[ffmpeg.git] / libavcodec / hevcdec.c
1 /*
2  * HEVC video decoder
3  *
4  * Copyright (C) 2012 - 2013 Guillaume Martres
5  * Copyright (C) 2012 - 2013 Mickael Raulet
6  * Copyright (C) 2012 - 2013 Gildas Cocherel
7  * Copyright (C) 2012 - 2013 Wassim Hamidouche
8  *
9  * This file is part of Libav.
10  *
11  * Libav is free software; you can redistribute it and/or
12  * modify it under the terms of the GNU Lesser General Public
13  * License as published by the Free Software Foundation; either
14  * version 2.1 of the License, or (at your option) any later version.
15  *
16  * Libav is distributed in the hope that it will be useful,
17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
19  * Lesser General Public License for more details.
20  *
21  * You should have received a copy of the GNU Lesser General Public
22  * License along with Libav; if not, write to the Free Software
23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24  */
25
26 #include "libavutil/attributes.h"
27 #include "libavutil/common.h"
28 #include "libavutil/display.h"
29 #include "libavutil/internal.h"
30 #include "libavutil/md5.h"
31 #include "libavutil/opt.h"
32 #include "libavutil/pixdesc.h"
33 #include "libavutil/stereo3d.h"
34
35 #include "bswapdsp.h"
36 #include "bytestream.h"
37 #include "cabac_functions.h"
38 #include "golomb_legacy.h"
39 #include "hevc.h"
40 #include "hevc_data.h"
41 #include "hevcdec.h"
42 #include "profiles.h"
43
44 const uint8_t ff_hevc_qpel_extra_before[4] = { 0, 3, 3, 3 };
45 const uint8_t ff_hevc_qpel_extra_after[4]  = { 0, 4, 4, 4 };
46 const uint8_t ff_hevc_qpel_extra[4]        = { 0, 7, 7, 7 };
47
48 static const uint8_t scan_1x1[1] = { 0 };
49
50 static const uint8_t horiz_scan2x2_x[4] = { 0, 1, 0, 1 };
51
52 static const uint8_t horiz_scan2x2_y[4] = { 0, 0, 1, 1 };
53
54 static const uint8_t horiz_scan4x4_x[16] = {
55     0, 1, 2, 3,
56     0, 1, 2, 3,
57     0, 1, 2, 3,
58     0, 1, 2, 3,
59 };
60
61 static const uint8_t horiz_scan4x4_y[16] = {
62     0, 0, 0, 0,
63     1, 1, 1, 1,
64     2, 2, 2, 2,
65     3, 3, 3, 3,
66 };
67
68 static const uint8_t horiz_scan8x8_inv[8][8] = {
69     {  0,  1,  2,  3, 16, 17, 18, 19, },
70     {  4,  5,  6,  7, 20, 21, 22, 23, },
71     {  8,  9, 10, 11, 24, 25, 26, 27, },
72     { 12, 13, 14, 15, 28, 29, 30, 31, },
73     { 32, 33, 34, 35, 48, 49, 50, 51, },
74     { 36, 37, 38, 39, 52, 53, 54, 55, },
75     { 40, 41, 42, 43, 56, 57, 58, 59, },
76     { 44, 45, 46, 47, 60, 61, 62, 63, },
77 };
78
79 static const uint8_t diag_scan2x2_x[4] = { 0, 0, 1, 1 };
80
81 static const uint8_t diag_scan2x2_y[4] = { 0, 1, 0, 1 };
82
83 static const uint8_t diag_scan2x2_inv[2][2] = {
84     { 0, 2, },
85     { 1, 3, },
86 };
87
88 static const uint8_t diag_scan4x4_inv[4][4] = {
89     { 0,  2,  5,  9, },
90     { 1,  4,  8, 12, },
91     { 3,  7, 11, 14, },
92     { 6, 10, 13, 15, },
93 };
94
95 static const uint8_t diag_scan8x8_inv[8][8] = {
96     {  0,  2,  5,  9, 14, 20, 27, 35, },
97     {  1,  4,  8, 13, 19, 26, 34, 42, },
98     {  3,  7, 12, 18, 25, 33, 41, 48, },
99     {  6, 11, 17, 24, 32, 40, 47, 53, },
100     { 10, 16, 23, 31, 39, 46, 52, 57, },
101     { 15, 22, 30, 38, 45, 51, 56, 60, },
102     { 21, 29, 37, 44, 50, 55, 59, 62, },
103     { 28, 36, 43, 49, 54, 58, 61, 63, },
104 };
105
106 /**
107  * NOTE: Each function hls_foo correspond to the function foo in the
108  * specification (HLS stands for High Level Syntax).
109  */
110
111 /**
112  * Section 5.7
113  */
114
115 /* free everything allocated  by pic_arrays_init() */
116 static void pic_arrays_free(HEVCContext *s)
117 {
118     av_freep(&s->sao);
119     av_freep(&s->deblock);
120
121     av_freep(&s->skip_flag);
122     av_freep(&s->tab_ct_depth);
123
124     av_freep(&s->tab_ipm);
125     av_freep(&s->cbf_luma);
126     av_freep(&s->is_pcm);
127
128     av_freep(&s->qp_y_tab);
129     av_freep(&s->tab_slice_address);
130     av_freep(&s->filter_slice_edges);
131
132     av_freep(&s->horizontal_bs);
133     av_freep(&s->vertical_bs);
134
135     av_buffer_pool_uninit(&s->tab_mvf_pool);
136     av_buffer_pool_uninit(&s->rpl_tab_pool);
137 }
138
139 /* allocate arrays that depend on frame dimensions */
140 static int pic_arrays_init(HEVCContext *s, const HEVCSPS *sps)
141 {
142     int log2_min_cb_size = sps->log2_min_cb_size;
143     int width            = sps->width;
144     int height           = sps->height;
145     int pic_size_in_ctb  = ((width  >> log2_min_cb_size) + 1) *
146                            ((height >> log2_min_cb_size) + 1);
147     int ctb_count        = sps->ctb_width * sps->ctb_height;
148     int min_pu_size      = sps->min_pu_width * sps->min_pu_height;
149
150     s->bs_width  = width  >> 3;
151     s->bs_height = height >> 3;
152
153     s->sao           = av_mallocz_array(ctb_count, sizeof(*s->sao));
154     s->deblock       = av_mallocz_array(ctb_count, sizeof(*s->deblock));
155     if (!s->sao || !s->deblock)
156         goto fail;
157
158     s->skip_flag    = av_malloc(pic_size_in_ctb);
159     s->tab_ct_depth = av_malloc(sps->min_cb_height * sps->min_cb_width);
160     if (!s->skip_flag || !s->tab_ct_depth)
161         goto fail;
162
163     s->cbf_luma = av_malloc(sps->min_tb_width * sps->min_tb_height);
164     s->tab_ipm  = av_mallocz(min_pu_size);
165     s->is_pcm   = av_malloc(min_pu_size);
166     if (!s->tab_ipm || !s->cbf_luma || !s->is_pcm)
167         goto fail;
168
169     s->filter_slice_edges = av_malloc(ctb_count);
170     s->tab_slice_address  = av_malloc(pic_size_in_ctb *
171                                       sizeof(*s->tab_slice_address));
172     s->qp_y_tab           = av_malloc(pic_size_in_ctb *
173                                       sizeof(*s->qp_y_tab));
174     if (!s->qp_y_tab || !s->filter_slice_edges || !s->tab_slice_address)
175         goto fail;
176
177     s->horizontal_bs = av_mallocz(2 * s->bs_width * (s->bs_height + 1));
178     s->vertical_bs   = av_mallocz(2 * s->bs_width * (s->bs_height + 1));
179     if (!s->horizontal_bs || !s->vertical_bs)
180         goto fail;
181
182     s->tab_mvf_pool = av_buffer_pool_init(min_pu_size * sizeof(MvField),
183                                           av_buffer_alloc);
184     s->rpl_tab_pool = av_buffer_pool_init(ctb_count * sizeof(RefPicListTab),
185                                           av_buffer_allocz);
186     if (!s->tab_mvf_pool || !s->rpl_tab_pool)
187         goto fail;
188
189     return 0;
190
191 fail:
192     pic_arrays_free(s);
193     return AVERROR(ENOMEM);
194 }
195
196 static void pred_weight_table(HEVCContext *s, GetBitContext *gb)
197 {
198     int i = 0;
199     int j = 0;
200     uint8_t luma_weight_l0_flag[16];
201     uint8_t chroma_weight_l0_flag[16];
202     uint8_t luma_weight_l1_flag[16];
203     uint8_t chroma_weight_l1_flag[16];
204
205     s->sh.luma_log2_weight_denom = av_clip(get_ue_golomb_long(gb), 0, 7);
206     if (s->ps.sps->chroma_format_idc != 0) {
207         int delta = get_se_golomb(gb);
208         s->sh.chroma_log2_weight_denom = av_clip(s->sh.luma_log2_weight_denom + delta, 0, 7);
209     }
210
211     for (i = 0; i < s->sh.nb_refs[L0]; i++) {
212         luma_weight_l0_flag[i] = get_bits1(gb);
213         if (!luma_weight_l0_flag[i]) {
214             s->sh.luma_weight_l0[i] = 1 << s->sh.luma_log2_weight_denom;
215             s->sh.luma_offset_l0[i] = 0;
216         }
217     }
218     if (s->ps.sps->chroma_format_idc != 0) { // FIXME: invert "if" and "for"
219         for (i = 0; i < s->sh.nb_refs[L0]; i++)
220             chroma_weight_l0_flag[i] = get_bits1(gb);
221     } else {
222         for (i = 0; i < s->sh.nb_refs[L0]; i++)
223             chroma_weight_l0_flag[i] = 0;
224     }
225     for (i = 0; i < s->sh.nb_refs[L0]; i++) {
226         if (luma_weight_l0_flag[i]) {
227             int delta_luma_weight_l0 = get_se_golomb(gb);
228             s->sh.luma_weight_l0[i] = (1 << s->sh.luma_log2_weight_denom) + delta_luma_weight_l0;
229             s->sh.luma_offset_l0[i] = get_se_golomb(gb);
230         }
231         if (chroma_weight_l0_flag[i]) {
232             for (j = 0; j < 2; j++) {
233                 int delta_chroma_weight_l0 = get_se_golomb(gb);
234                 int delta_chroma_offset_l0 = get_se_golomb(gb);
235                 s->sh.chroma_weight_l0[i][j] = (1 << s->sh.chroma_log2_weight_denom) + delta_chroma_weight_l0;
236                 s->sh.chroma_offset_l0[i][j] = av_clip((delta_chroma_offset_l0 - ((128 * s->sh.chroma_weight_l0[i][j])
237                                                                                     >> s->sh.chroma_log2_weight_denom) + 128), -128, 127);
238             }
239         } else {
240             s->sh.chroma_weight_l0[i][0] = 1 << s->sh.chroma_log2_weight_denom;
241             s->sh.chroma_offset_l0[i][0] = 0;
242             s->sh.chroma_weight_l0[i][1] = 1 << s->sh.chroma_log2_weight_denom;
243             s->sh.chroma_offset_l0[i][1] = 0;
244         }
245     }
246     if (s->sh.slice_type == HEVC_SLICE_B) {
247         for (i = 0; i < s->sh.nb_refs[L1]; i++) {
248             luma_weight_l1_flag[i] = get_bits1(gb);
249             if (!luma_weight_l1_flag[i]) {
250                 s->sh.luma_weight_l1[i] = 1 << s->sh.luma_log2_weight_denom;
251                 s->sh.luma_offset_l1[i] = 0;
252             }
253         }
254         if (s->ps.sps->chroma_format_idc != 0) {
255             for (i = 0; i < s->sh.nb_refs[L1]; i++)
256                 chroma_weight_l1_flag[i] = get_bits1(gb);
257         } else {
258             for (i = 0; i < s->sh.nb_refs[L1]; i++)
259                 chroma_weight_l1_flag[i] = 0;
260         }
261         for (i = 0; i < s->sh.nb_refs[L1]; i++) {
262             if (luma_weight_l1_flag[i]) {
263                 int delta_luma_weight_l1 = get_se_golomb(gb);
264                 s->sh.luma_weight_l1[i] = (1 << s->sh.luma_log2_weight_denom) + delta_luma_weight_l1;
265                 s->sh.luma_offset_l1[i] = get_se_golomb(gb);
266             }
267             if (chroma_weight_l1_flag[i]) {
268                 for (j = 0; j < 2; j++) {
269                     int delta_chroma_weight_l1 = get_se_golomb(gb);
270                     int delta_chroma_offset_l1 = get_se_golomb(gb);
271                     s->sh.chroma_weight_l1[i][j] = (1 << s->sh.chroma_log2_weight_denom) + delta_chroma_weight_l1;
272                     s->sh.chroma_offset_l1[i][j] = av_clip((delta_chroma_offset_l1 - ((128 * s->sh.chroma_weight_l1[i][j])
273                                                                                         >> s->sh.chroma_log2_weight_denom) + 128), -128, 127);
274                 }
275             } else {
276                 s->sh.chroma_weight_l1[i][0] = 1 << s->sh.chroma_log2_weight_denom;
277                 s->sh.chroma_offset_l1[i][0] = 0;
278                 s->sh.chroma_weight_l1[i][1] = 1 << s->sh.chroma_log2_weight_denom;
279                 s->sh.chroma_offset_l1[i][1] = 0;
280             }
281         }
282     }
283 }
284
285 static int decode_lt_rps(HEVCContext *s, LongTermRPS *rps, GetBitContext *gb)
286 {
287     const HEVCSPS *sps = s->ps.sps;
288     int max_poc_lsb    = 1 << sps->log2_max_poc_lsb;
289     int prev_delta_msb = 0;
290     unsigned int nb_sps = 0, nb_sh;
291     int i;
292
293     rps->nb_refs = 0;
294     if (!sps->long_term_ref_pics_present_flag)
295         return 0;
296
297     if (sps->num_long_term_ref_pics_sps > 0)
298         nb_sps = get_ue_golomb_long(gb);
299     nb_sh = get_ue_golomb_long(gb);
300
301     if (nb_sh + nb_sps > FF_ARRAY_ELEMS(rps->poc))
302         return AVERROR_INVALIDDATA;
303
304     rps->nb_refs = nb_sh + nb_sps;
305
306     for (i = 0; i < rps->nb_refs; i++) {
307         uint8_t delta_poc_msb_present;
308
309         if (i < nb_sps) {
310             uint8_t lt_idx_sps = 0;
311
312             if (sps->num_long_term_ref_pics_sps > 1)
313                 lt_idx_sps = get_bits(gb, av_ceil_log2(sps->num_long_term_ref_pics_sps));
314
315             rps->poc[i]  = sps->lt_ref_pic_poc_lsb_sps[lt_idx_sps];
316             rps->used[i] = sps->used_by_curr_pic_lt_sps_flag[lt_idx_sps];
317         } else {
318             rps->poc[i]  = get_bits(gb, sps->log2_max_poc_lsb);
319             rps->used[i] = get_bits1(gb);
320         }
321
322         delta_poc_msb_present = get_bits1(gb);
323         if (delta_poc_msb_present) {
324             int delta = get_ue_golomb_long(gb);
325
326             if (i && i != nb_sps)
327                 delta += prev_delta_msb;
328
329             rps->poc[i] += s->poc - delta * max_poc_lsb - s->sh.pic_order_cnt_lsb;
330             prev_delta_msb = delta;
331         }
332     }
333
334     return 0;
335 }
336
337 static void export_stream_params(AVCodecContext *avctx, const HEVCParamSets *ps,
338                                  const HEVCSPS *sps)
339 {
340     const HEVCVPS *vps = (const HEVCVPS*)ps->vps_list[sps->vps_id]->data;
341     const HEVCWindow *ow = &sps->output_window;
342     unsigned int num = 0, den = 0;
343
344     avctx->pix_fmt             = sps->pix_fmt;
345     avctx->coded_width         = sps->width;
346     avctx->coded_height        = sps->height;
347     avctx->width               = sps->width  - ow->left_offset - ow->right_offset;
348     avctx->height              = sps->height - ow->top_offset  - ow->bottom_offset;
349     avctx->has_b_frames        = sps->temporal_layer[sps->max_sub_layers - 1].num_reorder_pics;
350     avctx->profile             = sps->ptl.general_ptl.profile_idc;
351     avctx->level               = sps->ptl.general_ptl.level_idc;
352
353     ff_set_sar(avctx, sps->vui.sar);
354
355     if (sps->vui.video_signal_type_present_flag)
356         avctx->color_range = sps->vui.video_full_range_flag ? AVCOL_RANGE_JPEG
357                                                             : AVCOL_RANGE_MPEG;
358     else
359         avctx->color_range = AVCOL_RANGE_MPEG;
360
361     if (sps->vui.colour_description_present_flag) {
362         avctx->color_primaries = sps->vui.colour_primaries;
363         avctx->color_trc       = sps->vui.transfer_characteristic;
364         avctx->colorspace      = sps->vui.matrix_coeffs;
365     } else {
366         avctx->color_primaries = AVCOL_PRI_UNSPECIFIED;
367         avctx->color_trc       = AVCOL_TRC_UNSPECIFIED;
368         avctx->colorspace      = AVCOL_SPC_UNSPECIFIED;
369     }
370
371     if (vps->vps_timing_info_present_flag) {
372         num = vps->vps_num_units_in_tick;
373         den = vps->vps_time_scale;
374     } else if (sps->vui.vui_timing_info_present_flag) {
375         num = sps->vui.vui_num_units_in_tick;
376         den = sps->vui.vui_time_scale;
377     }
378
379     if (num != 0 && den != 0)
380         av_reduce(&avctx->framerate.den, &avctx->framerate.num,
381                   num, den, 1 << 30);
382 }
383
384 static enum AVPixelFormat get_format(HEVCContext *s, const HEVCSPS *sps)
385 {
386     #define HWACCEL_MAX (CONFIG_HEVC_DXVA2_HWACCEL + CONFIG_HEVC_D3D11VA_HWACCEL * 2 + \
387                          CONFIG_HEVC_VAAPI_HWACCEL + CONFIG_HEVC_VDPAU_HWACCEL + \
388                          CONFIG_HEVC_CUVID_HWACCEL)
389     enum AVPixelFormat pix_fmts[HWACCEL_MAX + 2], *fmt = pix_fmts;
390
391     if (sps->pix_fmt == AV_PIX_FMT_YUV420P || sps->pix_fmt == AV_PIX_FMT_YUVJ420P ||
392         sps->pix_fmt == AV_PIX_FMT_YUV420P10) {
393 #if CONFIG_HEVC_D3D11VA_HWACCEL
394         *fmt++ = AV_PIX_FMT_D3D11VA_VLD;
395         *fmt++ = AV_PIX_FMT_D3D11;
396 #endif
397 #if CONFIG_HEVC_DXVA2_HWACCEL
398         *fmt++ = AV_PIX_FMT_DXVA2_VLD;
399 #endif
400 #if CONFIG_HEVC_VAAPI_HWACCEL
401         *fmt++ = AV_PIX_FMT_VAAPI;
402 #endif
403 #if CONFIG_HEVC_CUVID_HWACCEL && HAVE_CUVIDDECODECREATEINFO_BITDEPTHMINUS8
404         *fmt++ = AV_PIX_FMT_CUDA;
405 #endif
406     }
407     if (sps->pix_fmt == AV_PIX_FMT_YUV420P || sps->pix_fmt == AV_PIX_FMT_YUVJ420P) {
408 #if CONFIG_HEVC_CUVID_HWACCEL && !HAVE_CUVIDDECODECREATEINFO_BITDEPTHMINUS8
409         *fmt++ = AV_PIX_FMT_CUDA;
410 #endif
411 #if CONFIG_HEVC_VDPAU_HWACCEL
412         *fmt++ = AV_PIX_FMT_VDPAU;
413 #endif
414     }
415
416     *fmt++ = sps->pix_fmt;
417     *fmt = AV_PIX_FMT_NONE;
418
419     return ff_get_format(s->avctx, pix_fmts);
420 }
421
422 static int set_sps(HEVCContext *s, const HEVCSPS *sps,
423                    enum AVPixelFormat pix_fmt)
424 {
425     int ret;
426
427     pic_arrays_free(s);
428     s->ps.sps = NULL;
429     s->ps.vps = NULL;
430
431     if (!sps)
432         return 0;
433
434     ret = pic_arrays_init(s, sps);
435     if (ret < 0)
436         goto fail;
437
438     export_stream_params(s->avctx, &s->ps, sps);
439
440     s->avctx->pix_fmt = pix_fmt;
441
442     ff_hevc_pred_init(&s->hpc,     sps->bit_depth);
443     ff_hevc_dsp_init (&s->hevcdsp, sps->bit_depth);
444     ff_videodsp_init (&s->vdsp,    sps->bit_depth);
445
446     if (sps->sao_enabled && !s->avctx->hwaccel) {
447         av_frame_unref(s->tmp_frame);
448         ret = ff_get_buffer(s->avctx, s->tmp_frame, AV_GET_BUFFER_FLAG_REF);
449         if (ret < 0)
450             goto fail;
451         s->frame = s->tmp_frame;
452     }
453
454     s->ps.sps = sps;
455     s->ps.vps = (HEVCVPS*) s->ps.vps_list[s->ps.sps->vps_id]->data;
456
457     return 0;
458
459 fail:
460     pic_arrays_free(s);
461     s->ps.sps = NULL;
462     return ret;
463 }
464
465 static int hls_slice_header(HEVCContext *s)
466 {
467     GetBitContext *gb = &s->HEVClc.gb;
468     SliceHeader *sh   = &s->sh;
469     int i, ret;
470
471     // Coded parameters
472     sh->first_slice_in_pic_flag = get_bits1(gb);
473     if ((IS_IDR(s) || IS_BLA(s)) && sh->first_slice_in_pic_flag) {
474         s->seq_decode = (s->seq_decode + 1) & 0xff;
475         s->max_ra     = INT_MAX;
476         if (IS_IDR(s))
477             ff_hevc_clear_refs(s);
478     }
479     if (IS_IRAP(s))
480         sh->no_output_of_prior_pics_flag = get_bits1(gb);
481
482     sh->pps_id = get_ue_golomb_long(gb);
483     if (sh->pps_id >= HEVC_MAX_PPS_COUNT || !s->ps.pps_list[sh->pps_id]) {
484         av_log(s->avctx, AV_LOG_ERROR, "PPS id out of range: %d\n", sh->pps_id);
485         return AVERROR_INVALIDDATA;
486     }
487     if (!sh->first_slice_in_pic_flag &&
488         s->ps.pps != (HEVCPPS*)s->ps.pps_list[sh->pps_id]->data) {
489         av_log(s->avctx, AV_LOG_ERROR, "PPS changed between slices.\n");
490         return AVERROR_INVALIDDATA;
491     }
492     s->ps.pps = (HEVCPPS*)s->ps.pps_list[sh->pps_id]->data;
493
494     if (s->ps.sps != (HEVCSPS*)s->ps.sps_list[s->ps.pps->sps_id]->data) {
495         const HEVCSPS *sps = (HEVCSPS*)s->ps.sps_list[s->ps.pps->sps_id]->data;
496         enum AVPixelFormat pix_fmt;
497
498         ff_hevc_clear_refs(s);
499
500         ret = set_sps(s, sps, sps->pix_fmt);
501         if (ret < 0)
502             return ret;
503
504         pix_fmt = get_format(s, sps);
505         if (pix_fmt < 0)
506             return pix_fmt;
507         s->avctx->pix_fmt = pix_fmt;
508
509         s->seq_decode = (s->seq_decode + 1) & 0xff;
510         s->max_ra     = INT_MAX;
511     }
512
513     sh->dependent_slice_segment_flag = 0;
514     if (!sh->first_slice_in_pic_flag) {
515         int slice_address_length;
516
517         if (s->ps.pps->dependent_slice_segments_enabled_flag)
518             sh->dependent_slice_segment_flag = get_bits1(gb);
519
520         slice_address_length = av_ceil_log2(s->ps.sps->ctb_width *
521                                             s->ps.sps->ctb_height);
522         sh->slice_segment_addr = slice_address_length ? get_bits(gb, slice_address_length) : 0;
523         if (sh->slice_segment_addr >= s->ps.sps->ctb_width * s->ps.sps->ctb_height) {
524             av_log(s->avctx, AV_LOG_ERROR,
525                    "Invalid slice segment address: %u.\n",
526                    sh->slice_segment_addr);
527             return AVERROR_INVALIDDATA;
528         }
529
530         if (!sh->dependent_slice_segment_flag) {
531             sh->slice_addr = sh->slice_segment_addr;
532             s->slice_idx++;
533         }
534     } else {
535         sh->slice_segment_addr = sh->slice_addr = 0;
536         s->slice_idx           = 0;
537         s->slice_initialized   = 0;
538     }
539
540     if (!sh->dependent_slice_segment_flag) {
541         s->slice_initialized = 0;
542
543         for (i = 0; i < s->ps.pps->num_extra_slice_header_bits; i++)
544             skip_bits(gb, 1);  // slice_reserved_undetermined_flag[]
545
546         sh->slice_type = get_ue_golomb_long(gb);
547         if (!(sh->slice_type == HEVC_SLICE_I ||
548               sh->slice_type == HEVC_SLICE_P ||
549               sh->slice_type == HEVC_SLICE_B)) {
550             av_log(s->avctx, AV_LOG_ERROR, "Unknown slice type: %d.\n",
551                    sh->slice_type);
552             return AVERROR_INVALIDDATA;
553         }
554         if (IS_IRAP(s) && sh->slice_type != HEVC_SLICE_I) {
555             av_log(s->avctx, AV_LOG_ERROR, "Inter slices in an IRAP frame.\n");
556             return AVERROR_INVALIDDATA;
557         }
558
559         // when flag is not present, picture is inferred to be output
560         sh->pic_output_flag = 1;
561         if (s->ps.pps->output_flag_present_flag)
562             sh->pic_output_flag = get_bits1(gb);
563
564         if (s->ps.sps->separate_colour_plane_flag)
565             sh->colour_plane_id = get_bits(gb, 2);
566
567         if (!IS_IDR(s)) {
568             int poc, pos;
569
570             sh->pic_order_cnt_lsb = get_bits(gb, s->ps.sps->log2_max_poc_lsb);
571             poc = ff_hevc_compute_poc(s, sh->pic_order_cnt_lsb);
572             if (!sh->first_slice_in_pic_flag && poc != s->poc) {
573                 av_log(s->avctx, AV_LOG_WARNING,
574                        "Ignoring POC change between slices: %d -> %d\n", s->poc, poc);
575                 if (s->avctx->err_recognition & AV_EF_EXPLODE)
576                     return AVERROR_INVALIDDATA;
577                 poc = s->poc;
578             }
579             s->poc = poc;
580
581             sh->short_term_ref_pic_set_sps_flag = get_bits1(gb);
582             pos = get_bits_left(gb);
583             if (!sh->short_term_ref_pic_set_sps_flag) {
584                 ret = ff_hevc_decode_short_term_rps(gb, s->avctx, &sh->slice_rps, s->ps.sps, 1);
585                 if (ret < 0)
586                     return ret;
587
588                 sh->short_term_rps = &sh->slice_rps;
589             } else {
590                 int numbits, rps_idx;
591
592                 if (!s->ps.sps->nb_st_rps) {
593                     av_log(s->avctx, AV_LOG_ERROR, "No ref lists in the SPS.\n");
594                     return AVERROR_INVALIDDATA;
595                 }
596
597                 numbits = av_ceil_log2(s->ps.sps->nb_st_rps);
598                 rps_idx = numbits > 0 ? get_bits(gb, numbits) : 0;
599                 sh->short_term_rps = &s->ps.sps->st_rps[rps_idx];
600             }
601             sh->short_term_ref_pic_set_size = pos - get_bits_left(gb);
602
603             pos = get_bits_left(gb);
604             ret = decode_lt_rps(s, &sh->long_term_rps, gb);
605             if (ret < 0) {
606                 av_log(s->avctx, AV_LOG_WARNING, "Invalid long term RPS.\n");
607                 if (s->avctx->err_recognition & AV_EF_EXPLODE)
608                     return AVERROR_INVALIDDATA;
609             }
610             sh->long_term_ref_pic_set_size = pos - get_bits_left(gb);
611
612             if (s->ps.sps->sps_temporal_mvp_enabled_flag)
613                 sh->slice_temporal_mvp_enabled_flag = get_bits1(gb);
614             else
615                 sh->slice_temporal_mvp_enabled_flag = 0;
616         } else {
617             s->sh.short_term_rps = NULL;
618             s->poc               = 0;
619         }
620
621         /* 8.3.1 */
622         if (s->temporal_id == 0 &&
623             s->nal_unit_type != HEVC_NAL_TRAIL_N &&
624             s->nal_unit_type != HEVC_NAL_TSA_N   &&
625             s->nal_unit_type != HEVC_NAL_STSA_N  &&
626             s->nal_unit_type != HEVC_NAL_RADL_N  &&
627             s->nal_unit_type != HEVC_NAL_RADL_R  &&
628             s->nal_unit_type != HEVC_NAL_RASL_N  &&
629             s->nal_unit_type != HEVC_NAL_RASL_R)
630             s->pocTid0 = s->poc;
631
632         if (s->ps.sps->sao_enabled) {
633             sh->slice_sample_adaptive_offset_flag[0] = get_bits1(gb);
634             sh->slice_sample_adaptive_offset_flag[1] =
635             sh->slice_sample_adaptive_offset_flag[2] = get_bits1(gb);
636         } else {
637             sh->slice_sample_adaptive_offset_flag[0] = 0;
638             sh->slice_sample_adaptive_offset_flag[1] = 0;
639             sh->slice_sample_adaptive_offset_flag[2] = 0;
640         }
641
642         sh->nb_refs[L0] = sh->nb_refs[L1] = 0;
643         if (sh->slice_type == HEVC_SLICE_P || sh->slice_type == HEVC_SLICE_B) {
644             int nb_refs;
645
646             sh->nb_refs[L0] = s->ps.pps->num_ref_idx_l0_default_active;
647             if (sh->slice_type == HEVC_SLICE_B)
648                 sh->nb_refs[L1] = s->ps.pps->num_ref_idx_l1_default_active;
649
650             if (get_bits1(gb)) { // num_ref_idx_active_override_flag
651                 sh->nb_refs[L0] = get_ue_golomb_long(gb) + 1;
652                 if (sh->slice_type == HEVC_SLICE_B)
653                     sh->nb_refs[L1] = get_ue_golomb_long(gb) + 1;
654             }
655             if (sh->nb_refs[L0] > HEVC_MAX_REFS || sh->nb_refs[L1] > HEVC_MAX_REFS) {
656                 av_log(s->avctx, AV_LOG_ERROR, "Too many refs: %d/%d.\n",
657                        sh->nb_refs[L0], sh->nb_refs[L1]);
658                 return AVERROR_INVALIDDATA;
659             }
660
661             sh->rpl_modification_flag[0] = 0;
662             sh->rpl_modification_flag[1] = 0;
663             nb_refs = ff_hevc_frame_nb_refs(s);
664             if (!nb_refs) {
665                 av_log(s->avctx, AV_LOG_ERROR, "Zero refs for a frame with P or B slices.\n");
666                 return AVERROR_INVALIDDATA;
667             }
668
669             if (s->ps.pps->lists_modification_present_flag && nb_refs > 1) {
670                 sh->rpl_modification_flag[0] = get_bits1(gb);
671                 if (sh->rpl_modification_flag[0]) {
672                     for (i = 0; i < sh->nb_refs[L0]; i++)
673                         sh->list_entry_lx[0][i] = get_bits(gb, av_ceil_log2(nb_refs));
674                 }
675
676                 if (sh->slice_type == HEVC_SLICE_B) {
677                     sh->rpl_modification_flag[1] = get_bits1(gb);
678                     if (sh->rpl_modification_flag[1] == 1)
679                         for (i = 0; i < sh->nb_refs[L1]; i++)
680                             sh->list_entry_lx[1][i] = get_bits(gb, av_ceil_log2(nb_refs));
681                 }
682             }
683
684             if (sh->slice_type == HEVC_SLICE_B)
685                 sh->mvd_l1_zero_flag = get_bits1(gb);
686
687             if (s->ps.pps->cabac_init_present_flag)
688                 sh->cabac_init_flag = get_bits1(gb);
689             else
690                 sh->cabac_init_flag = 0;
691
692             sh->collocated_ref_idx = 0;
693             if (sh->slice_temporal_mvp_enabled_flag) {
694                 sh->collocated_list = L0;
695                 if (sh->slice_type == HEVC_SLICE_B)
696                     sh->collocated_list = !get_bits1(gb);
697
698                 if (sh->nb_refs[sh->collocated_list] > 1) {
699                     sh->collocated_ref_idx = get_ue_golomb_long(gb);
700                     if (sh->collocated_ref_idx >= sh->nb_refs[sh->collocated_list]) {
701                         av_log(s->avctx, AV_LOG_ERROR,
702                                "Invalid collocated_ref_idx: %d.\n",
703                                sh->collocated_ref_idx);
704                         return AVERROR_INVALIDDATA;
705                     }
706                 }
707             }
708
709             if ((s->ps.pps->weighted_pred_flag   && sh->slice_type == HEVC_SLICE_P) ||
710                 (s->ps.pps->weighted_bipred_flag && sh->slice_type == HEVC_SLICE_B)) {
711                 pred_weight_table(s, gb);
712             }
713
714             sh->max_num_merge_cand = 5 - get_ue_golomb_long(gb);
715             if (sh->max_num_merge_cand < 1 || sh->max_num_merge_cand > 5) {
716                 av_log(s->avctx, AV_LOG_ERROR,
717                        "Invalid number of merging MVP candidates: %d.\n",
718                        sh->max_num_merge_cand);
719                 return AVERROR_INVALIDDATA;
720             }
721         }
722
723         sh->slice_qp_delta = get_se_golomb(gb);
724
725         if (s->ps.pps->pic_slice_level_chroma_qp_offsets_present_flag) {
726             sh->slice_cb_qp_offset = get_se_golomb(gb);
727             sh->slice_cr_qp_offset = get_se_golomb(gb);
728         } else {
729             sh->slice_cb_qp_offset = 0;
730             sh->slice_cr_qp_offset = 0;
731         }
732
733         if (s->ps.pps->deblocking_filter_control_present_flag) {
734             int deblocking_filter_override_flag = 0;
735
736             if (s->ps.pps->deblocking_filter_override_enabled_flag)
737                 deblocking_filter_override_flag = get_bits1(gb);
738
739             if (deblocking_filter_override_flag) {
740                 sh->disable_deblocking_filter_flag = get_bits1(gb);
741                 if (!sh->disable_deblocking_filter_flag) {
742                     sh->beta_offset = get_se_golomb(gb) * 2;
743                     sh->tc_offset   = get_se_golomb(gb) * 2;
744                 }
745             } else {
746                 sh->disable_deblocking_filter_flag = s->ps.pps->disable_dbf;
747                 sh->beta_offset                    = s->ps.pps->beta_offset;
748                 sh->tc_offset                      = s->ps.pps->tc_offset;
749             }
750         } else {
751             sh->disable_deblocking_filter_flag = 0;
752             sh->beta_offset                    = 0;
753             sh->tc_offset                      = 0;
754         }
755
756         if (s->ps.pps->seq_loop_filter_across_slices_enabled_flag &&
757             (sh->slice_sample_adaptive_offset_flag[0] ||
758              sh->slice_sample_adaptive_offset_flag[1] ||
759              !sh->disable_deblocking_filter_flag)) {
760             sh->slice_loop_filter_across_slices_enabled_flag = get_bits1(gb);
761         } else {
762             sh->slice_loop_filter_across_slices_enabled_flag = s->ps.pps->seq_loop_filter_across_slices_enabled_flag;
763         }
764     } else if (!s->slice_initialized) {
765         av_log(s->avctx, AV_LOG_ERROR, "Independent slice segment missing.\n");
766         return AVERROR_INVALIDDATA;
767     }
768
769     sh->num_entry_point_offsets = 0;
770     if (s->ps.pps->tiles_enabled_flag || s->ps.pps->entropy_coding_sync_enabled_flag) {
771         sh->num_entry_point_offsets = get_ue_golomb_long(gb);
772         if (sh->num_entry_point_offsets > 0) {
773             int offset_len = get_ue_golomb_long(gb) + 1;
774
775             for (i = 0; i < sh->num_entry_point_offsets; i++)
776                 skip_bits(gb, offset_len);
777         }
778     }
779
780     if (s->ps.pps->slice_header_extension_present_flag) {
781         unsigned int length = get_ue_golomb_long(gb);
782         for (i = 0; i < length; i++)
783             skip_bits(gb, 8);  // slice_header_extension_data_byte
784     }
785
786     // Inferred parameters
787     sh->slice_qp = 26 + s->ps.pps->pic_init_qp_minus26 + sh->slice_qp_delta;
788     if (sh->slice_qp > 51 ||
789         sh->slice_qp < -s->ps.sps->qp_bd_offset) {
790         av_log(s->avctx, AV_LOG_ERROR,
791                "The slice_qp %d is outside the valid range "
792                "[%d, 51].\n",
793                sh->slice_qp,
794                -s->ps.sps->qp_bd_offset);
795         return AVERROR_INVALIDDATA;
796     }
797
798     sh->slice_ctb_addr_rs = sh->slice_segment_addr;
799
800     if (!s->sh.slice_ctb_addr_rs && s->sh.dependent_slice_segment_flag) {
801         av_log(s->avctx, AV_LOG_ERROR, "Impossible slice segment.\n");
802         return AVERROR_INVALIDDATA;
803     }
804
805     s->HEVClc.first_qp_group = !s->sh.dependent_slice_segment_flag;
806
807     if (!s->ps.pps->cu_qp_delta_enabled_flag)
808         s->HEVClc.qp_y = FFUMOD(s->sh.slice_qp + 52 + 2 * s->ps.sps->qp_bd_offset,
809                                 52 + s->ps.sps->qp_bd_offset) - s->ps.sps->qp_bd_offset;
810
811     s->slice_initialized = 1;
812
813     return 0;
814 }
815
816 #define CTB(tab, x, y) ((tab)[(y) * s->ps.sps->ctb_width + (x)])
817
818 #define SET_SAO(elem, value)                            \
819 do {                                                    \
820     if (!sao_merge_up_flag && !sao_merge_left_flag)     \
821         sao->elem = value;                              \
822     else if (sao_merge_left_flag)                       \
823         sao->elem = CTB(s->sao, rx-1, ry).elem;         \
824     else if (sao_merge_up_flag)                         \
825         sao->elem = CTB(s->sao, rx, ry-1).elem;         \
826     else                                                \
827         sao->elem = 0;                                  \
828 } while (0)
829
830 static void hls_sao_param(HEVCContext *s, int rx, int ry)
831 {
832     HEVCLocalContext *lc    = &s->HEVClc;
833     int sao_merge_left_flag = 0;
834     int sao_merge_up_flag   = 0;
835     int shift               = s->ps.sps->bit_depth - FFMIN(s->ps.sps->bit_depth, 10);
836     SAOParams *sao          = &CTB(s->sao, rx, ry);
837     int c_idx, i;
838
839     if (s->sh.slice_sample_adaptive_offset_flag[0] ||
840         s->sh.slice_sample_adaptive_offset_flag[1]) {
841         if (rx > 0) {
842             if (lc->ctb_left_flag)
843                 sao_merge_left_flag = ff_hevc_sao_merge_flag_decode(s);
844         }
845         if (ry > 0 && !sao_merge_left_flag) {
846             if (lc->ctb_up_flag)
847                 sao_merge_up_flag = ff_hevc_sao_merge_flag_decode(s);
848         }
849     }
850
851     for (c_idx = 0; c_idx < 3; c_idx++) {
852         if (!s->sh.slice_sample_adaptive_offset_flag[c_idx]) {
853             sao->type_idx[c_idx] = SAO_NOT_APPLIED;
854             continue;
855         }
856
857         if (c_idx == 2) {
858             sao->type_idx[2] = sao->type_idx[1];
859             sao->eo_class[2] = sao->eo_class[1];
860         } else {
861             SET_SAO(type_idx[c_idx], ff_hevc_sao_type_idx_decode(s));
862         }
863
864         if (sao->type_idx[c_idx] == SAO_NOT_APPLIED)
865             continue;
866
867         for (i = 0; i < 4; i++)
868             SET_SAO(offset_abs[c_idx][i], ff_hevc_sao_offset_abs_decode(s));
869
870         if (sao->type_idx[c_idx] == SAO_BAND) {
871             for (i = 0; i < 4; i++) {
872                 if (sao->offset_abs[c_idx][i]) {
873                     SET_SAO(offset_sign[c_idx][i],
874                             ff_hevc_sao_offset_sign_decode(s));
875                 } else {
876                     sao->offset_sign[c_idx][i] = 0;
877                 }
878             }
879             SET_SAO(band_position[c_idx], ff_hevc_sao_band_position_decode(s));
880         } else if (c_idx != 2) {
881             SET_SAO(eo_class[c_idx], ff_hevc_sao_eo_class_decode(s));
882         }
883
884         // Inferred parameters
885         sao->offset_val[c_idx][0] = 0;
886         for (i = 0; i < 4; i++) {
887             sao->offset_val[c_idx][i + 1] = sao->offset_abs[c_idx][i] << shift;
888             if (sao->type_idx[c_idx] == SAO_EDGE) {
889                 if (i > 1)
890                     sao->offset_val[c_idx][i + 1] = -sao->offset_val[c_idx][i + 1];
891             } else if (sao->offset_sign[c_idx][i]) {
892                 sao->offset_val[c_idx][i + 1] = -sao->offset_val[c_idx][i + 1];
893             }
894         }
895     }
896 }
897
898 #undef SET_SAO
899 #undef CTB
900
901 static void hls_residual_coding(HEVCContext *s, int x0, int y0,
902                                 int log2_trafo_size, enum ScanType scan_idx,
903                                 int c_idx)
904 {
905 #define GET_COORD(offset, n)                                    \
906     do {                                                        \
907         x_c = (scan_x_cg[offset >> 4] << 2) + scan_x_off[n];    \
908         y_c = (scan_y_cg[offset >> 4] << 2) + scan_y_off[n];    \
909     } while (0)
910     HEVCLocalContext *lc    = &s->HEVClc;
911     int transform_skip_flag = 0;
912
913     int last_significant_coeff_x, last_significant_coeff_y;
914     int last_scan_pos;
915     int n_end;
916     int num_coeff    = 0;
917     int greater1_ctx = 1;
918
919     int num_last_subset;
920     int x_cg_last_sig, y_cg_last_sig;
921
922     const uint8_t *scan_x_cg, *scan_y_cg, *scan_x_off, *scan_y_off;
923
924     ptrdiff_t stride = s->frame->linesize[c_idx];
925     int hshift       = s->ps.sps->hshift[c_idx];
926     int vshift       = s->ps.sps->vshift[c_idx];
927     uint8_t *dst     = &s->frame->data[c_idx][(y0 >> vshift) * stride +
928                                               ((x0 >> hshift) << s->ps.sps->pixel_shift)];
929     LOCAL_ALIGNED_32(int16_t, coeffs, [MAX_TB_SIZE * MAX_TB_SIZE]);
930     LOCAL_ALIGNED_8(uint8_t, significant_coeff_group_flag, [8], [8]);
931
932     int trafo_size = 1 << log2_trafo_size;
933     int i, qp, shift, add, scale, scale_m;
934     static const uint8_t level_scale[] = { 40, 45, 51, 57, 64, 72 };
935     const uint8_t *scale_matrix;
936     uint8_t dc_scale;
937
938     memset(coeffs, 0, sizeof(int16_t) * MAX_TB_SIZE * MAX_TB_SIZE);
939     memset(significant_coeff_group_flag, 0, sizeof(uint8_t) * 8 * 8);
940     // Derive QP for dequant
941     if (!lc->cu.cu_transquant_bypass_flag) {
942         static const int qp_c[] = {
943             29, 30, 31, 32, 33, 33, 34, 34, 35, 35, 36, 36, 37, 37
944         };
945
946         static const uint8_t rem6[51 + 2 * 6 + 1] = {
947             0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2,
948             3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5,
949             0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3,
950         };
951
952         static const uint8_t div6[51 + 2 * 6 + 1] = {
953             0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2,  3,  3,  3,
954             3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 6, 6, 6,  6,  6,  6,
955             7, 7, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10,
956         };
957         int qp_y = lc->qp_y;
958
959         if (c_idx == 0) {
960             qp = qp_y + s->ps.sps->qp_bd_offset;
961         } else {
962             int qp_i, offset;
963
964             if (c_idx == 1)
965                 offset = s->ps.pps->cb_qp_offset + s->sh.slice_cb_qp_offset;
966             else
967                 offset = s->ps.pps->cr_qp_offset + s->sh.slice_cr_qp_offset;
968
969             qp_i = av_clip(qp_y + offset, -s->ps.sps->qp_bd_offset, 57);
970             if (qp_i < 30)
971                 qp = qp_i;
972             else if (qp_i > 43)
973                 qp = qp_i - 6;
974             else
975                 qp = qp_c[qp_i - 30];
976
977             qp += s->ps.sps->qp_bd_offset;
978         }
979
980         shift    = s->ps.sps->bit_depth + log2_trafo_size - 5;
981         add      = 1 << (shift - 1);
982         scale    = level_scale[rem6[qp]] << (div6[qp]);
983         scale_m  = 16; // default when no custom scaling lists.
984         dc_scale = 16;
985
986         if (s->ps.sps->scaling_list_enable_flag) {
987             const ScalingList *sl = s->ps.pps->scaling_list_data_present_flag ?
988                                     &s->ps.pps->scaling_list : &s->ps.sps->scaling_list;
989             int matrix_id = lc->cu.pred_mode != MODE_INTRA;
990
991             if (log2_trafo_size != 5)
992                 matrix_id = 3 * matrix_id + c_idx;
993
994             scale_matrix = sl->sl[log2_trafo_size - 2][matrix_id];
995             if (log2_trafo_size >= 4)
996                 dc_scale = sl->sl_dc[log2_trafo_size - 4][matrix_id];
997         }
998     }
999
1000     if (s->ps.pps->transform_skip_enabled_flag &&
1001         !lc->cu.cu_transquant_bypass_flag   &&
1002         log2_trafo_size == 2) {
1003         transform_skip_flag = ff_hevc_transform_skip_flag_decode(s, c_idx);
1004     }
1005
1006     last_significant_coeff_x =
1007         ff_hevc_last_significant_coeff_x_prefix_decode(s, c_idx, log2_trafo_size);
1008     last_significant_coeff_y =
1009         ff_hevc_last_significant_coeff_y_prefix_decode(s, c_idx, log2_trafo_size);
1010
1011     if (last_significant_coeff_x > 3) {
1012         int suffix = ff_hevc_last_significant_coeff_suffix_decode(s, last_significant_coeff_x);
1013         last_significant_coeff_x = (1 << ((last_significant_coeff_x >> 1) - 1)) *
1014                                    (2 + (last_significant_coeff_x & 1)) +
1015                                    suffix;
1016     }
1017
1018     if (last_significant_coeff_y > 3) {
1019         int suffix = ff_hevc_last_significant_coeff_suffix_decode(s, last_significant_coeff_y);
1020         last_significant_coeff_y = (1 << ((last_significant_coeff_y >> 1) - 1)) *
1021                                    (2 + (last_significant_coeff_y & 1)) +
1022                                    suffix;
1023     }
1024
1025     if (scan_idx == SCAN_VERT)
1026         FFSWAP(int, last_significant_coeff_x, last_significant_coeff_y);
1027
1028     x_cg_last_sig = last_significant_coeff_x >> 2;
1029     y_cg_last_sig = last_significant_coeff_y >> 2;
1030
1031     switch (scan_idx) {
1032     case SCAN_DIAG: {
1033         int last_x_c = last_significant_coeff_x & 3;
1034         int last_y_c = last_significant_coeff_y & 3;
1035
1036         scan_x_off = ff_hevc_diag_scan4x4_x;
1037         scan_y_off = ff_hevc_diag_scan4x4_y;
1038         num_coeff  = diag_scan4x4_inv[last_y_c][last_x_c];
1039         if (trafo_size == 4) {
1040             scan_x_cg = scan_1x1;
1041             scan_y_cg = scan_1x1;
1042         } else if (trafo_size == 8) {
1043             num_coeff += diag_scan2x2_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1044             scan_x_cg  = diag_scan2x2_x;
1045             scan_y_cg  = diag_scan2x2_y;
1046         } else if (trafo_size == 16) {
1047             num_coeff += diag_scan4x4_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1048             scan_x_cg  = ff_hevc_diag_scan4x4_x;
1049             scan_y_cg  = ff_hevc_diag_scan4x4_y;
1050         } else { // trafo_size == 32
1051             num_coeff += diag_scan8x8_inv[y_cg_last_sig][x_cg_last_sig] << 4;
1052             scan_x_cg  = ff_hevc_diag_scan8x8_x;
1053             scan_y_cg  = ff_hevc_diag_scan8x8_y;
1054         }
1055         break;
1056     }
1057     case SCAN_HORIZ:
1058         scan_x_cg  = horiz_scan2x2_x;
1059         scan_y_cg  = horiz_scan2x2_y;
1060         scan_x_off = horiz_scan4x4_x;
1061         scan_y_off = horiz_scan4x4_y;
1062         num_coeff  = horiz_scan8x8_inv[last_significant_coeff_y][last_significant_coeff_x];
1063         break;
1064     default: //SCAN_VERT
1065         scan_x_cg  = horiz_scan2x2_y;
1066         scan_y_cg  = horiz_scan2x2_x;
1067         scan_x_off = horiz_scan4x4_y;
1068         scan_y_off = horiz_scan4x4_x;
1069         num_coeff  = horiz_scan8x8_inv[last_significant_coeff_x][last_significant_coeff_y];
1070         break;
1071     }
1072     num_coeff++;
1073     num_last_subset = (num_coeff - 1) >> 4;
1074
1075     for (i = num_last_subset; i >= 0; i--) {
1076         int n, m;
1077         int x_cg, y_cg, x_c, y_c;
1078         int implicit_non_zero_coeff = 0;
1079         int64_t trans_coeff_level;
1080         int prev_sig = 0;
1081         int offset   = i << 4;
1082
1083         uint8_t significant_coeff_flag_idx[16];
1084         uint8_t nb_significant_coeff_flag = 0;
1085
1086         x_cg = scan_x_cg[i];
1087         y_cg = scan_y_cg[i];
1088
1089         if (i < num_last_subset && i > 0) {
1090             int ctx_cg = 0;
1091             if (x_cg < (1 << (log2_trafo_size - 2)) - 1)
1092                 ctx_cg += significant_coeff_group_flag[x_cg + 1][y_cg];
1093             if (y_cg < (1 << (log2_trafo_size - 2)) - 1)
1094                 ctx_cg += significant_coeff_group_flag[x_cg][y_cg + 1];
1095
1096             significant_coeff_group_flag[x_cg][y_cg] =
1097                 ff_hevc_significant_coeff_group_flag_decode(s, c_idx, ctx_cg);
1098             implicit_non_zero_coeff = 1;
1099         } else {
1100             significant_coeff_group_flag[x_cg][y_cg] =
1101                 ((x_cg == x_cg_last_sig && y_cg == y_cg_last_sig) ||
1102                  (x_cg == 0 && y_cg == 0));
1103         }
1104
1105         last_scan_pos = num_coeff - offset - 1;
1106
1107         if (i == num_last_subset) {
1108             n_end                         = last_scan_pos - 1;
1109             significant_coeff_flag_idx[0] = last_scan_pos;
1110             nb_significant_coeff_flag     = 1;
1111         } else {
1112             n_end = 15;
1113         }
1114
1115         if (x_cg < ((1 << log2_trafo_size) - 1) >> 2)
1116             prev_sig = significant_coeff_group_flag[x_cg + 1][y_cg];
1117         if (y_cg < ((1 << log2_trafo_size) - 1) >> 2)
1118             prev_sig += significant_coeff_group_flag[x_cg][y_cg + 1] << 1;
1119
1120         for (n = n_end; n >= 0; n--) {
1121             GET_COORD(offset, n);
1122
1123             if (significant_coeff_group_flag[x_cg][y_cg] &&
1124                 (n > 0 || implicit_non_zero_coeff == 0)) {
1125                 if (ff_hevc_significant_coeff_flag_decode(s, c_idx, x_c, y_c,
1126                                                           log2_trafo_size,
1127                                                           scan_idx,
1128                                                           prev_sig) == 1) {
1129                     significant_coeff_flag_idx[nb_significant_coeff_flag] = n;
1130                     nb_significant_coeff_flag++;
1131                     implicit_non_zero_coeff = 0;
1132                 }
1133             } else {
1134                 int last_cg = (x_c == (x_cg << 2) && y_c == (y_cg << 2));
1135                 if (last_cg && implicit_non_zero_coeff && significant_coeff_group_flag[x_cg][y_cg]) {
1136                     significant_coeff_flag_idx[nb_significant_coeff_flag] = n;
1137                     nb_significant_coeff_flag++;
1138                 }
1139             }
1140         }
1141
1142         n_end = nb_significant_coeff_flag;
1143
1144         if (n_end) {
1145             int first_nz_pos_in_cg = 16;
1146             int last_nz_pos_in_cg = -1;
1147             int c_rice_param = 0;
1148             int first_greater1_coeff_idx = -1;
1149             uint8_t coeff_abs_level_greater1_flag[16] = { 0 };
1150             uint16_t coeff_sign_flag;
1151             int sum_abs = 0;
1152             int sign_hidden = 0;
1153
1154             // initialize first elem of coeff_bas_level_greater1_flag
1155             int ctx_set = (i > 0 && c_idx == 0) ? 2 : 0;
1156
1157             if (!(i == num_last_subset) && greater1_ctx == 0)
1158                 ctx_set++;
1159             greater1_ctx      = 1;
1160             last_nz_pos_in_cg = significant_coeff_flag_idx[0];
1161
1162             for (m = 0; m < (n_end > 8 ? 8 : n_end); m++) {
1163                 int n_idx = significant_coeff_flag_idx[m];
1164                 int inc   = (ctx_set << 2) + greater1_ctx;
1165                 coeff_abs_level_greater1_flag[n_idx] =
1166                     ff_hevc_coeff_abs_level_greater1_flag_decode(s, c_idx, inc);
1167                 if (coeff_abs_level_greater1_flag[n_idx]) {
1168                     greater1_ctx = 0;
1169                 } else if (greater1_ctx > 0 && greater1_ctx < 3) {
1170                     greater1_ctx++;
1171                 }
1172
1173                 if (coeff_abs_level_greater1_flag[n_idx] &&
1174                     first_greater1_coeff_idx == -1)
1175                     first_greater1_coeff_idx = n_idx;
1176             }
1177             first_nz_pos_in_cg = significant_coeff_flag_idx[n_end - 1];
1178             sign_hidden        = last_nz_pos_in_cg - first_nz_pos_in_cg >= 4 &&
1179                                  !lc->cu.cu_transquant_bypass_flag;
1180
1181             if (first_greater1_coeff_idx != -1) {
1182                 coeff_abs_level_greater1_flag[first_greater1_coeff_idx] += ff_hevc_coeff_abs_level_greater2_flag_decode(s, c_idx, ctx_set);
1183             }
1184             if (!s->ps.pps->sign_data_hiding_flag || !sign_hidden) {
1185                 coeff_sign_flag = ff_hevc_coeff_sign_flag(s, nb_significant_coeff_flag) << (16 - nb_significant_coeff_flag);
1186             } else {
1187                 coeff_sign_flag = ff_hevc_coeff_sign_flag(s, nb_significant_coeff_flag - 1) << (16 - (nb_significant_coeff_flag - 1));
1188             }
1189
1190             for (m = 0; m < n_end; m++) {
1191                 n = significant_coeff_flag_idx[m];
1192                 GET_COORD(offset, n);
1193                 trans_coeff_level = 1 + coeff_abs_level_greater1_flag[n];
1194                 if (trans_coeff_level == ((m < 8) ?
1195                                           ((n == first_greater1_coeff_idx) ? 3 : 2) : 1)) {
1196                     trans_coeff_level += ff_hevc_coeff_abs_level_remaining(s, trans_coeff_level, c_rice_param);
1197                     if ((trans_coeff_level) > (3 * (1 << c_rice_param)))
1198                         c_rice_param = FFMIN(c_rice_param + 1, 4);
1199                 }
1200                 if (s->ps.pps->sign_data_hiding_flag && sign_hidden) {
1201                     sum_abs += trans_coeff_level;
1202                     if (n == first_nz_pos_in_cg && ((sum_abs & 1) == 1))
1203                         trans_coeff_level = -trans_coeff_level;
1204                 }
1205                 if (coeff_sign_flag >> 15)
1206                     trans_coeff_level = -trans_coeff_level;
1207                 coeff_sign_flag <<= 1;
1208                 if (!lc->cu.cu_transquant_bypass_flag) {
1209                     if (s->ps.sps->scaling_list_enable_flag) {
1210                         if (y_c || x_c || log2_trafo_size < 4) {
1211                             int pos;
1212                             switch (log2_trafo_size) {
1213                             case 3:  pos = (y_c        << 3) +  x_c;       break;
1214                             case 4:  pos = ((y_c >> 1) << 3) + (x_c >> 1); break;
1215                             case 5:  pos = ((y_c >> 2) << 3) + (x_c >> 2); break;
1216                             default: pos = (y_c        << 2) +  x_c;
1217                             }
1218                             scale_m = scale_matrix[pos];
1219                         } else {
1220                             scale_m = dc_scale;
1221                         }
1222                     }
1223                     trans_coeff_level = (trans_coeff_level * (int64_t)scale * (int64_t)scale_m + add) >> shift;
1224                     if(trans_coeff_level < 0) {
1225                         if((~trans_coeff_level) & 0xFffffffffff8000)
1226                             trans_coeff_level = -32768;
1227                     } else {
1228                         if (trans_coeff_level & 0xffffffffffff8000)
1229                             trans_coeff_level = 32767;
1230                     }
1231                 }
1232                 coeffs[y_c * trafo_size + x_c] = trans_coeff_level;
1233             }
1234         }
1235     }
1236
1237     if (!lc->cu.cu_transquant_bypass_flag) {
1238         if (transform_skip_flag)
1239             s->hevcdsp.dequant(coeffs);
1240         else if (lc->cu.pred_mode == MODE_INTRA && c_idx == 0 &&
1241                  log2_trafo_size == 2)
1242             s->hevcdsp.transform_4x4_luma(coeffs);
1243         else {
1244             int max_xy = FFMAX(last_significant_coeff_x, last_significant_coeff_y);
1245             if (max_xy == 0)
1246                 s->hevcdsp.idct_dc[log2_trafo_size - 2](coeffs);
1247             else {
1248                 int col_limit = last_significant_coeff_x + last_significant_coeff_y + 4;
1249                 if (max_xy < 4)
1250                     col_limit = FFMIN(4, col_limit);
1251                 else if (max_xy < 8)
1252                     col_limit = FFMIN(8, col_limit);
1253                 else if (max_xy < 12)
1254                     col_limit = FFMIN(24, col_limit);
1255                 s->hevcdsp.idct[log2_trafo_size - 2](coeffs, col_limit);
1256             }
1257         }
1258     }
1259     s->hevcdsp.add_residual[log2_trafo_size - 2](dst, coeffs, stride);
1260 }
1261
1262 static int hls_transform_unit(HEVCContext *s, int x0, int y0,
1263                               int xBase, int yBase, int cb_xBase, int cb_yBase,
1264                               int log2_cb_size, int log2_trafo_size,
1265                               int blk_idx, int cbf_luma, int cbf_cb, int cbf_cr)
1266 {
1267     HEVCLocalContext *lc = &s->HEVClc;
1268
1269     if (lc->cu.pred_mode == MODE_INTRA) {
1270         int trafo_size = 1 << log2_trafo_size;
1271         ff_hevc_set_neighbour_available(s, x0, y0, trafo_size, trafo_size);
1272
1273         s->hpc.intra_pred[log2_trafo_size - 2](s, x0, y0, 0);
1274         if (log2_trafo_size > 2) {
1275             trafo_size = trafo_size << (s->ps.sps->hshift[1] - 1);
1276             ff_hevc_set_neighbour_available(s, x0, y0, trafo_size, trafo_size);
1277             s->hpc.intra_pred[log2_trafo_size - 3](s, x0, y0, 1);
1278             s->hpc.intra_pred[log2_trafo_size - 3](s, x0, y0, 2);
1279         } else if (blk_idx == 3) {
1280             trafo_size = trafo_size << s->ps.sps->hshift[1];
1281             ff_hevc_set_neighbour_available(s, xBase, yBase,
1282                                             trafo_size, trafo_size);
1283             s->hpc.intra_pred[log2_trafo_size - 2](s, xBase, yBase, 1);
1284             s->hpc.intra_pred[log2_trafo_size - 2](s, xBase, yBase, 2);
1285         }
1286     }
1287
1288     if (cbf_luma || cbf_cb || cbf_cr) {
1289         int scan_idx   = SCAN_DIAG;
1290         int scan_idx_c = SCAN_DIAG;
1291
1292         if (s->ps.pps->cu_qp_delta_enabled_flag && !lc->tu.is_cu_qp_delta_coded) {
1293             lc->tu.cu_qp_delta = ff_hevc_cu_qp_delta_abs(s);
1294             if (lc->tu.cu_qp_delta != 0)
1295                 if (ff_hevc_cu_qp_delta_sign_flag(s) == 1)
1296                     lc->tu.cu_qp_delta = -lc->tu.cu_qp_delta;
1297             lc->tu.is_cu_qp_delta_coded = 1;
1298
1299             if (lc->tu.cu_qp_delta < -(26 + s->ps.sps->qp_bd_offset / 2) ||
1300                 lc->tu.cu_qp_delta >  (25 + s->ps.sps->qp_bd_offset / 2)) {
1301                 av_log(s->avctx, AV_LOG_ERROR,
1302                        "The cu_qp_delta %d is outside the valid range "
1303                        "[%d, %d].\n",
1304                        lc->tu.cu_qp_delta,
1305                        -(26 + s->ps.sps->qp_bd_offset / 2),
1306                         (25 + s->ps.sps->qp_bd_offset / 2));
1307                 return AVERROR_INVALIDDATA;
1308             }
1309
1310             ff_hevc_set_qPy(s, x0, y0, cb_xBase, cb_yBase, log2_cb_size);
1311         }
1312
1313         if (lc->cu.pred_mode == MODE_INTRA && log2_trafo_size < 4) {
1314             if (lc->tu.cur_intra_pred_mode >= 6 &&
1315                 lc->tu.cur_intra_pred_mode <= 14) {
1316                 scan_idx = SCAN_VERT;
1317             } else if (lc->tu.cur_intra_pred_mode >= 22 &&
1318                        lc->tu.cur_intra_pred_mode <= 30) {
1319                 scan_idx = SCAN_HORIZ;
1320             }
1321
1322             if (lc->pu.intra_pred_mode_c >=  6 &&
1323                 lc->pu.intra_pred_mode_c <= 14) {
1324                 scan_idx_c = SCAN_VERT;
1325             } else if (lc->pu.intra_pred_mode_c >= 22 &&
1326                        lc->pu.intra_pred_mode_c <= 30) {
1327                 scan_idx_c = SCAN_HORIZ;
1328             }
1329         }
1330
1331         if (cbf_luma)
1332             hls_residual_coding(s, x0, y0, log2_trafo_size, scan_idx, 0);
1333         if (log2_trafo_size > 2) {
1334             if (cbf_cb)
1335                 hls_residual_coding(s, x0, y0, log2_trafo_size - 1, scan_idx_c, 1);
1336             if (cbf_cr)
1337                 hls_residual_coding(s, x0, y0, log2_trafo_size - 1, scan_idx_c, 2);
1338         } else if (blk_idx == 3) {
1339             if (cbf_cb)
1340                 hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 1);
1341             if (cbf_cr)
1342                 hls_residual_coding(s, xBase, yBase, log2_trafo_size, scan_idx_c, 2);
1343         }
1344     }
1345     return 0;
1346 }
1347
1348 static void set_deblocking_bypass(HEVCContext *s, int x0, int y0, int log2_cb_size)
1349 {
1350     int cb_size          = 1 << log2_cb_size;
1351     int log2_min_pu_size = s->ps.sps->log2_min_pu_size;
1352
1353     int min_pu_width     = s->ps.sps->min_pu_width;
1354     int x_end = FFMIN(x0 + cb_size, s->ps.sps->width);
1355     int y_end = FFMIN(y0 + cb_size, s->ps.sps->height);
1356     int i, j;
1357
1358     for (j = (y0 >> log2_min_pu_size); j < (y_end >> log2_min_pu_size); j++)
1359         for (i = (x0 >> log2_min_pu_size); i < (x_end >> log2_min_pu_size); i++)
1360             s->is_pcm[i + j * min_pu_width] = 2;
1361 }
1362
1363 static int hls_transform_tree(HEVCContext *s, int x0, int y0,
1364                               int xBase, int yBase, int cb_xBase, int cb_yBase,
1365                               int log2_cb_size, int log2_trafo_size,
1366                               int trafo_depth, int blk_idx,
1367                               int cbf_cb, int cbf_cr)
1368 {
1369     HEVCLocalContext *lc = &s->HEVClc;
1370     uint8_t split_transform_flag;
1371     int ret;
1372
1373     if (lc->cu.intra_split_flag) {
1374         if (trafo_depth == 1)
1375             lc->tu.cur_intra_pred_mode = lc->pu.intra_pred_mode[blk_idx];
1376     } else {
1377         lc->tu.cur_intra_pred_mode = lc->pu.intra_pred_mode[0];
1378     }
1379
1380     if (log2_trafo_size <= s->ps.sps->log2_max_trafo_size &&
1381         log2_trafo_size >  s->ps.sps->log2_min_tb_size    &&
1382         trafo_depth     < lc->cu.max_trafo_depth       &&
1383         !(lc->cu.intra_split_flag && trafo_depth == 0)) {
1384         split_transform_flag = ff_hevc_split_transform_flag_decode(s, log2_trafo_size);
1385     } else {
1386         int inter_split = s->ps.sps->max_transform_hierarchy_depth_inter == 0 &&
1387                           lc->cu.pred_mode == MODE_INTER &&
1388                           lc->cu.part_mode != PART_2Nx2N &&
1389                           trafo_depth == 0;
1390
1391         split_transform_flag = log2_trafo_size > s->ps.sps->log2_max_trafo_size ||
1392                                (lc->cu.intra_split_flag && trafo_depth == 0) ||
1393                                inter_split;
1394     }
1395
1396     if (log2_trafo_size > 2 && (trafo_depth == 0 || cbf_cb))
1397         cbf_cb = ff_hevc_cbf_cb_cr_decode(s, trafo_depth);
1398     else if (log2_trafo_size > 2 || trafo_depth == 0)
1399         cbf_cb = 0;
1400     if (log2_trafo_size > 2 && (trafo_depth == 0 || cbf_cr))
1401         cbf_cr = ff_hevc_cbf_cb_cr_decode(s, trafo_depth);
1402     else if (log2_trafo_size > 2 || trafo_depth == 0)
1403         cbf_cr = 0;
1404
1405     if (split_transform_flag) {
1406         const int trafo_size_split = 1 << (log2_trafo_size - 1);
1407         const int x1 = x0 + trafo_size_split;
1408         const int y1 = y0 + trafo_size_split;
1409
1410 #define SUBDIVIDE(x, y, idx)                                                    \
1411 do {                                                                            \
1412     ret = hls_transform_tree(s, x, y, x0, y0, cb_xBase, cb_yBase, log2_cb_size, \
1413                              log2_trafo_size - 1, trafo_depth + 1, idx,         \
1414                              cbf_cb, cbf_cr);                                   \
1415     if (ret < 0)                                                                \
1416         return ret;                                                             \
1417 } while (0)
1418
1419         SUBDIVIDE(x0, y0, 0);
1420         SUBDIVIDE(x1, y0, 1);
1421         SUBDIVIDE(x0, y1, 2);
1422         SUBDIVIDE(x1, y1, 3);
1423
1424 #undef SUBDIVIDE
1425     } else {
1426         int min_tu_size      = 1 << s->ps.sps->log2_min_tb_size;
1427         int log2_min_tu_size = s->ps.sps->log2_min_tb_size;
1428         int min_tu_width     = s->ps.sps->min_tb_width;
1429         int cbf_luma         = 1;
1430
1431         if (lc->cu.pred_mode == MODE_INTRA || trafo_depth != 0 ||
1432             cbf_cb || cbf_cr)
1433             cbf_luma = ff_hevc_cbf_luma_decode(s, trafo_depth);
1434
1435         ret = hls_transform_unit(s, x0, y0, xBase, yBase, cb_xBase, cb_yBase,
1436                                  log2_cb_size, log2_trafo_size,
1437                                  blk_idx, cbf_luma, cbf_cb, cbf_cr);
1438         if (ret < 0)
1439             return ret;
1440         // TODO: store cbf_luma somewhere else
1441         if (cbf_luma) {
1442             int i, j;
1443             for (i = 0; i < (1 << log2_trafo_size); i += min_tu_size)
1444                 for (j = 0; j < (1 << log2_trafo_size); j += min_tu_size) {
1445                     int x_tu = (x0 + j) >> log2_min_tu_size;
1446                     int y_tu = (y0 + i) >> log2_min_tu_size;
1447                     s->cbf_luma[y_tu * min_tu_width + x_tu] = 1;
1448                 }
1449         }
1450         if (!s->sh.disable_deblocking_filter_flag) {
1451             ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_trafo_size);
1452             if (s->ps.pps->transquant_bypass_enable_flag &&
1453                 lc->cu.cu_transquant_bypass_flag)
1454                 set_deblocking_bypass(s, x0, y0, log2_trafo_size);
1455         }
1456     }
1457     return 0;
1458 }
1459
1460 static int hls_pcm_sample(HEVCContext *s, int x0, int y0, int log2_cb_size)
1461 {
1462     //TODO: non-4:2:0 support
1463     HEVCLocalContext *lc = &s->HEVClc;
1464     GetBitContext gb;
1465     int cb_size   = 1 << log2_cb_size;
1466     ptrdiff_t stride0 = s->frame->linesize[0];
1467     ptrdiff_t stride1 = s->frame->linesize[1];
1468     ptrdiff_t stride2 = s->frame->linesize[2];
1469     uint8_t *dst0 = &s->frame->data[0][y0 * stride0 + (x0 << s->ps.sps->pixel_shift)];
1470     uint8_t *dst1 = &s->frame->data[1][(y0 >> s->ps.sps->vshift[1]) * stride1 + ((x0 >> s->ps.sps->hshift[1]) << s->ps.sps->pixel_shift)];
1471     uint8_t *dst2 = &s->frame->data[2][(y0 >> s->ps.sps->vshift[2]) * stride2 + ((x0 >> s->ps.sps->hshift[2]) << s->ps.sps->pixel_shift)];
1472
1473     int length         = cb_size * cb_size * s->ps.sps->pcm.bit_depth + ((cb_size * cb_size) >> 1) * s->ps.sps->pcm.bit_depth_chroma;
1474     const uint8_t *pcm = skip_bytes(&lc->cc, (length + 7) >> 3);
1475     int ret;
1476
1477     if (!s->sh.disable_deblocking_filter_flag)
1478         ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
1479
1480     ret = init_get_bits(&gb, pcm, length);
1481     if (ret < 0)
1482         return ret;
1483
1484     s->hevcdsp.put_pcm(dst0, stride0, cb_size,     &gb, s->ps.sps->pcm.bit_depth);
1485     s->hevcdsp.put_pcm(dst1, stride1, cb_size / 2, &gb, s->ps.sps->pcm.bit_depth_chroma);
1486     s->hevcdsp.put_pcm(dst2, stride2, cb_size / 2, &gb, s->ps.sps->pcm.bit_depth_chroma);
1487     return 0;
1488 }
1489
1490 static void hls_mvd_coding(HEVCContext *s, int x0, int y0, int log2_cb_size)
1491 {
1492     HEVCLocalContext *lc = &s->HEVClc;
1493     int x = ff_hevc_abs_mvd_greater0_flag_decode(s);
1494     int y = ff_hevc_abs_mvd_greater0_flag_decode(s);
1495
1496     if (x)
1497         x += ff_hevc_abs_mvd_greater1_flag_decode(s);
1498     if (y)
1499         y += ff_hevc_abs_mvd_greater1_flag_decode(s);
1500
1501     switch (x) {
1502     case 2: lc->pu.mvd.x = ff_hevc_mvd_decode(s);           break;
1503     case 1: lc->pu.mvd.x = ff_hevc_mvd_sign_flag_decode(s); break;
1504     case 0: lc->pu.mvd.x = 0;                               break;
1505     }
1506
1507     switch (y) {
1508     case 2: lc->pu.mvd.y = ff_hevc_mvd_decode(s);           break;
1509     case 1: lc->pu.mvd.y = ff_hevc_mvd_sign_flag_decode(s); break;
1510     case 0: lc->pu.mvd.y = 0;                               break;
1511     }
1512 }
1513
1514 /**
1515  * 8.5.3.2.2.1 Luma sample interpolation process
1516  *
1517  * @param s HEVC decoding context
1518  * @param dst target buffer for block data at block position
1519  * @param dststride stride of the dst buffer
1520  * @param ref reference picture buffer at origin (0, 0)
1521  * @param mv motion vector (relative to block position) to get pixel data from
1522  * @param x_off horizontal position of block from origin (0, 0)
1523  * @param y_off vertical position of block from origin (0, 0)
1524  * @param block_w width of block
1525  * @param block_h height of block
1526  */
1527 static void luma_mc(HEVCContext *s, int16_t *dst, ptrdiff_t dststride,
1528                     AVFrame *ref, const Mv *mv, int x_off, int y_off,
1529                     int block_w, int block_h, int pred_idx)
1530 {
1531     HEVCLocalContext *lc = &s->HEVClc;
1532     uint8_t *src         = ref->data[0];
1533     ptrdiff_t srcstride  = ref->linesize[0];
1534     int pic_width        = s->ps.sps->width;
1535     int pic_height       = s->ps.sps->height;
1536
1537     int mx         = mv->x & 3;
1538     int my         = mv->y & 3;
1539     int extra_left = ff_hevc_qpel_extra_before[mx];
1540     int extra_top  = ff_hevc_qpel_extra_before[my];
1541
1542     x_off += mv->x >> 2;
1543     y_off += mv->y >> 2;
1544     src   += y_off * srcstride + (x_off * (1 << s->ps.sps->pixel_shift));
1545
1546     if (x_off < extra_left || y_off < extra_top ||
1547         x_off >= pic_width - block_w - ff_hevc_qpel_extra_after[mx] ||
1548         y_off >= pic_height - block_h - ff_hevc_qpel_extra_after[my]) {
1549         const ptrdiff_t edge_emu_stride = EDGE_EMU_BUFFER_STRIDE << s->ps.sps->pixel_shift;
1550         int offset = extra_top * srcstride + (extra_left << s->ps.sps->pixel_shift);
1551         int buf_offset = extra_top *
1552                          edge_emu_stride + (extra_left << s->ps.sps->pixel_shift);
1553
1554         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src - offset,
1555                                  edge_emu_stride, srcstride,
1556                                  block_w + ff_hevc_qpel_extra[mx],
1557                                  block_h + ff_hevc_qpel_extra[my],
1558                                  x_off - extra_left, y_off - extra_top,
1559                                  pic_width, pic_height);
1560         src = lc->edge_emu_buffer + buf_offset;
1561         srcstride = edge_emu_stride;
1562     }
1563     s->hevcdsp.put_hevc_qpel[!!my][!!mx][pred_idx](dst, dststride, src, srcstride,
1564                                                    block_h, mx, my, lc->mc_buffer);
1565 }
1566
1567 /**
1568  * 8.5.3.2.2.2 Chroma sample interpolation process
1569  *
1570  * @param s HEVC decoding context
1571  * @param dst1 target buffer for block data at block position (U plane)
1572  * @param dst2 target buffer for block data at block position (V plane)
1573  * @param dststride stride of the dst1 and dst2 buffers
1574  * @param ref reference picture buffer at origin (0, 0)
1575  * @param mv motion vector (relative to block position) to get pixel data from
1576  * @param x_off horizontal position of block from origin (0, 0)
1577  * @param y_off vertical position of block from origin (0, 0)
1578  * @param block_w width of block
1579  * @param block_h height of block
1580  */
1581 static void chroma_mc(HEVCContext *s, int16_t *dst1, int16_t *dst2,
1582                       ptrdiff_t dststride, AVFrame *ref, const Mv *mv,
1583                       int x_off, int y_off, int block_w, int block_h, int pred_idx)
1584 {
1585     HEVCLocalContext *lc = &s->HEVClc;
1586     uint8_t *src1        = ref->data[1];
1587     uint8_t *src2        = ref->data[2];
1588     ptrdiff_t src1stride = ref->linesize[1];
1589     ptrdiff_t src2stride = ref->linesize[2];
1590     int pic_width        = s->ps.sps->width >> 1;
1591     int pic_height       = s->ps.sps->height >> 1;
1592
1593     int mx = mv->x & 7;
1594     int my = mv->y & 7;
1595
1596     x_off += mv->x >> 3;
1597     y_off += mv->y >> 3;
1598     src1  += y_off * src1stride + (x_off * (1 << s->ps.sps->pixel_shift));
1599     src2  += y_off * src2stride + (x_off * (1 << s->ps.sps->pixel_shift));
1600
1601     if (x_off < EPEL_EXTRA_BEFORE || y_off < EPEL_EXTRA_AFTER ||
1602         x_off >= pic_width - block_w - EPEL_EXTRA_AFTER ||
1603         y_off >= pic_height - block_h - EPEL_EXTRA_AFTER) {
1604         const ptrdiff_t edge_emu_stride = EDGE_EMU_BUFFER_STRIDE << s->ps.sps->pixel_shift;
1605         int offset1 = EPEL_EXTRA_BEFORE * (src1stride + (1 << s->ps.sps->pixel_shift));
1606         int buf_offset1 = EPEL_EXTRA_BEFORE *
1607                           (edge_emu_stride + (1 << s->ps.sps->pixel_shift));
1608         int offset2 = EPEL_EXTRA_BEFORE * (src2stride + (1 << s->ps.sps->pixel_shift));
1609         int buf_offset2 = EPEL_EXTRA_BEFORE *
1610                           (edge_emu_stride + (1 << s->ps.sps->pixel_shift));
1611
1612         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src1 - offset1,
1613                                  edge_emu_stride, src1stride,
1614                                  block_w + EPEL_EXTRA, block_h + EPEL_EXTRA,
1615                                  x_off - EPEL_EXTRA_BEFORE,
1616                                  y_off - EPEL_EXTRA_BEFORE,
1617                                  pic_width, pic_height);
1618
1619         src1 = lc->edge_emu_buffer + buf_offset1;
1620         src1stride = edge_emu_stride;
1621         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst1, dststride, src1, src1stride,
1622                                                        block_h, mx, my, lc->mc_buffer);
1623
1624         s->vdsp.emulated_edge_mc(lc->edge_emu_buffer, src2 - offset2,
1625                                  edge_emu_stride, src2stride,
1626                                  block_w + EPEL_EXTRA, block_h + EPEL_EXTRA,
1627                                  x_off - EPEL_EXTRA_BEFORE,
1628                                  y_off - EPEL_EXTRA_BEFORE,
1629                                  pic_width, pic_height);
1630         src2 = lc->edge_emu_buffer + buf_offset2;
1631         src2stride = edge_emu_stride;
1632
1633         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst2, dststride, src2, src2stride,
1634                                                        block_h, mx, my, lc->mc_buffer);
1635     } else {
1636         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst1, dststride, src1, src1stride,
1637                                                        block_h, mx, my, lc->mc_buffer);
1638         s->hevcdsp.put_hevc_epel[!!my][!!mx][pred_idx](dst2, dststride, src2, src2stride,
1639                                                        block_h, mx, my, lc->mc_buffer);
1640     }
1641 }
1642
1643 static void hevc_await_progress(HEVCContext *s, HEVCFrame *ref,
1644                                 const Mv *mv, int y0, int height)
1645 {
1646     int y = (mv->y >> 2) + y0 + height + 9;
1647     ff_thread_await_progress(&ref->tf, y, 0);
1648 }
1649
1650 static void hevc_luma_mv_mpv_mode(HEVCContext *s, int x0, int y0, int nPbW,
1651                                   int nPbH, int log2_cb_size, int part_idx,
1652                                   int merge_idx, MvField *mv)
1653 {
1654     HEVCLocalContext *lc             = &s->HEVClc;
1655     enum InterPredIdc inter_pred_idc = PRED_L0;
1656     int mvp_flag;
1657
1658     ff_hevc_set_neighbour_available(s, x0, y0, nPbW, nPbH);
1659     if (s->sh.slice_type == HEVC_SLICE_B)
1660         inter_pred_idc = ff_hevc_inter_pred_idc_decode(s, nPbW, nPbH);
1661
1662     if (inter_pred_idc != PRED_L1) {
1663         if (s->sh.nb_refs[L0])
1664             mv->ref_idx[0]= ff_hevc_ref_idx_lx_decode(s, s->sh.nb_refs[L0]);
1665
1666         mv->pred_flag[0] = 1;
1667         hls_mvd_coding(s, x0, y0, 0);
1668         mvp_flag = ff_hevc_mvp_lx_flag_decode(s);
1669         ff_hevc_luma_mv_mvp_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1670                                  part_idx, merge_idx, mv, mvp_flag, 0);
1671         mv->mv[0].x += lc->pu.mvd.x;
1672         mv->mv[0].y += lc->pu.mvd.y;
1673     }
1674
1675     if (inter_pred_idc != PRED_L0) {
1676         if (s->sh.nb_refs[L1])
1677             mv->ref_idx[1]= ff_hevc_ref_idx_lx_decode(s, s->sh.nb_refs[L1]);
1678
1679         if (s->sh.mvd_l1_zero_flag == 1 && inter_pred_idc == PRED_BI) {
1680             AV_ZERO32(&lc->pu.mvd);
1681         } else {
1682             hls_mvd_coding(s, x0, y0, 1);
1683         }
1684
1685         mv->pred_flag[1] = 1;
1686         mvp_flag = ff_hevc_mvp_lx_flag_decode(s);
1687         ff_hevc_luma_mv_mvp_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1688                                  part_idx, merge_idx, mv, mvp_flag, 1);
1689         mv->mv[1].x += lc->pu.mvd.x;
1690         mv->mv[1].y += lc->pu.mvd.y;
1691     }
1692 }
1693
1694 static void hls_prediction_unit(HEVCContext *s, int x0, int y0,
1695                                 int nPbW, int nPbH,
1696                                 int log2_cb_size, int partIdx)
1697 {
1698     static const int pred_indices[] = {
1699         [4] = 0, [8] = 1, [12] = 2, [16] = 3, [24] = 4, [32] = 5, [48] = 6, [64] = 7,
1700     };
1701     const int pred_idx = pred_indices[nPbW];
1702
1703 #define POS(c_idx, x, y)                                                              \
1704     &s->frame->data[c_idx][((y) >> s->ps.sps->vshift[c_idx]) * s->frame->linesize[c_idx] + \
1705                            (((x) >> s->ps.sps->hshift[c_idx]) << s->ps.sps->pixel_shift)]
1706     HEVCLocalContext *lc = &s->HEVClc;
1707     int merge_idx = 0;
1708     struct MvField current_mv = {{{ 0 }}};
1709
1710     int min_pu_width = s->ps.sps->min_pu_width;
1711     int weighted_pred = (s->sh.slice_type == HEVC_SLICE_P && s->ps.pps->weighted_pred_flag) ||
1712                         (s->sh.slice_type == HEVC_SLICE_B && s->ps.pps->weighted_bipred_flag);
1713
1714     MvField *tab_mvf = s->ref->tab_mvf;
1715     RefPicList  *refPicList = s->ref->refPicList;
1716     HEVCFrame *ref0, *ref1;
1717
1718     ptrdiff_t tmpstride = MAX_PB_SIZE * sizeof(int16_t);
1719
1720     uint8_t *dst0 = POS(0, x0, y0);
1721     uint8_t *dst1 = POS(1, x0, y0);
1722     uint8_t *dst2 = POS(2, x0, y0);
1723     int log2_min_cb_size = s->ps.sps->log2_min_cb_size;
1724     int min_cb_width     = s->ps.sps->min_cb_width;
1725     int x_cb             = x0 >> log2_min_cb_size;
1726     int y_cb             = y0 >> log2_min_cb_size;
1727     int x_pu, y_pu;
1728     int i, j;
1729
1730     int skip_flag = SAMPLE_CTB(s->skip_flag, x_cb, y_cb);
1731
1732     if (!skip_flag)
1733         lc->pu.merge_flag = ff_hevc_merge_flag_decode(s);
1734
1735     if (skip_flag || lc->pu.merge_flag) {
1736         if (s->sh.max_num_merge_cand > 1)
1737             merge_idx = ff_hevc_merge_idx_decode(s);
1738         else
1739             merge_idx = 0;
1740
1741         ff_hevc_luma_mv_merge_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1742                                    partIdx, merge_idx, &current_mv);
1743     } else {
1744         hevc_luma_mv_mpv_mode(s, x0, y0, nPbW, nPbH, log2_cb_size,
1745                               partIdx, merge_idx, &current_mv);
1746     }
1747
1748     x_pu = x0 >> s->ps.sps->log2_min_pu_size;
1749     y_pu = y0 >> s->ps.sps->log2_min_pu_size;
1750
1751     for (j = 0; j < nPbH >> s->ps.sps->log2_min_pu_size; j++)
1752         for (i = 0; i < nPbW >> s->ps.sps->log2_min_pu_size; i++)
1753             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i] = current_mv;
1754
1755     if (current_mv.pred_flag[0]) {
1756         ref0 = refPicList[0].ref[current_mv.ref_idx[0]];
1757         if (!ref0)
1758             return;
1759         hevc_await_progress(s, ref0, &current_mv.mv[0], y0, nPbH);
1760     }
1761     if (current_mv.pred_flag[1]) {
1762         ref1 = refPicList[1].ref[current_mv.ref_idx[1]];
1763         if (!ref1)
1764             return;
1765         hevc_await_progress(s, ref1, &current_mv.mv[1], y0, nPbH);
1766     }
1767
1768     if (current_mv.pred_flag[0] && !current_mv.pred_flag[1]) {
1769         LOCAL_ALIGNED_16(int16_t,  tmp, [MAX_PB_SIZE * MAX_PB_SIZE]);
1770         LOCAL_ALIGNED_16(int16_t, tmp2, [MAX_PB_SIZE * MAX_PB_SIZE]);
1771
1772         luma_mc(s, tmp, tmpstride, ref0->frame,
1773                 &current_mv.mv[0], x0, y0, nPbW, nPbH, pred_idx);
1774
1775         if (weighted_pred) {
1776             s->hevcdsp.weighted_pred[pred_idx](s->sh.luma_log2_weight_denom,
1777                                                s->sh.luma_weight_l0[current_mv.ref_idx[0]],
1778                                                s->sh.luma_offset_l0[current_mv.ref_idx[0]],
1779                                                dst0, s->frame->linesize[0], tmp,
1780                                                tmpstride, nPbH);
1781         } else {
1782             s->hevcdsp.put_unweighted_pred[pred_idx](dst0, s->frame->linesize[0], tmp, tmpstride, nPbH);
1783         }
1784         chroma_mc(s, tmp, tmp2, tmpstride, ref0->frame,
1785                   &current_mv.mv[0], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1786
1787         if (weighted_pred) {
1788             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1789                                                       s->sh.chroma_weight_l0[current_mv.ref_idx[0]][0],
1790                                                       s->sh.chroma_offset_l0[current_mv.ref_idx[0]][0],
1791                                                       dst1, s->frame->linesize[1], tmp, tmpstride,
1792                                                       nPbH / 2);
1793             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1794                                                       s->sh.chroma_weight_l0[current_mv.ref_idx[0]][1],
1795                                                       s->sh.chroma_offset_l0[current_mv.ref_idx[0]][1],
1796                                                       dst2, s->frame->linesize[2], tmp2, tmpstride,
1797                                                       nPbH / 2);
1798         } else {
1799             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst1, s->frame->linesize[1], tmp,  tmpstride, nPbH / 2);
1800             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst2, s->frame->linesize[2], tmp2, tmpstride, nPbH / 2);
1801         }
1802     } else if (!current_mv.pred_flag[0] && current_mv.pred_flag[1]) {
1803         LOCAL_ALIGNED_16(int16_t, tmp,  [MAX_PB_SIZE * MAX_PB_SIZE]);
1804         LOCAL_ALIGNED_16(int16_t, tmp2, [MAX_PB_SIZE * MAX_PB_SIZE]);
1805
1806         luma_mc(s, tmp, tmpstride, ref1->frame,
1807                 &current_mv.mv[1], x0, y0, nPbW, nPbH, pred_idx);
1808
1809         if (weighted_pred) {
1810             s->hevcdsp.weighted_pred[pred_idx](s->sh.luma_log2_weight_denom,
1811                                                s->sh.luma_weight_l1[current_mv.ref_idx[1]],
1812                                                s->sh.luma_offset_l1[current_mv.ref_idx[1]],
1813                                                dst0, s->frame->linesize[0], tmp, tmpstride,
1814                                                nPbH);
1815         } else {
1816             s->hevcdsp.put_unweighted_pred[pred_idx](dst0, s->frame->linesize[0], tmp, tmpstride, nPbH);
1817         }
1818
1819         chroma_mc(s, tmp, tmp2, tmpstride, ref1->frame,
1820                   &current_mv.mv[1], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1821
1822         if (weighted_pred) {
1823             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1824                                                       s->sh.chroma_weight_l1[current_mv.ref_idx[1]][0],
1825                                                       s->sh.chroma_offset_l1[current_mv.ref_idx[1]][0],
1826                                                       dst1, s->frame->linesize[1], tmp, tmpstride, nPbH/2);
1827             s->hevcdsp.weighted_pred_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1828                                                       s->sh.chroma_weight_l1[current_mv.ref_idx[1]][1],
1829                                                       s->sh.chroma_offset_l1[current_mv.ref_idx[1]][1],
1830                                                       dst2, s->frame->linesize[2], tmp2, tmpstride, nPbH/2);
1831         } else {
1832             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst1, s->frame->linesize[1], tmp,  tmpstride, nPbH / 2);
1833             s->hevcdsp.put_unweighted_pred_chroma[pred_idx](dst2, s->frame->linesize[2], tmp2, tmpstride, nPbH / 2);
1834         }
1835     } else if (current_mv.pred_flag[0] && current_mv.pred_flag[1]) {
1836         LOCAL_ALIGNED_16(int16_t, tmp,  [MAX_PB_SIZE * MAX_PB_SIZE]);
1837         LOCAL_ALIGNED_16(int16_t, tmp2, [MAX_PB_SIZE * MAX_PB_SIZE]);
1838         LOCAL_ALIGNED_16(int16_t, tmp3, [MAX_PB_SIZE * MAX_PB_SIZE]);
1839         LOCAL_ALIGNED_16(int16_t, tmp4, [MAX_PB_SIZE * MAX_PB_SIZE]);
1840
1841         luma_mc(s, tmp, tmpstride, ref0->frame,
1842                 &current_mv.mv[0], x0, y0, nPbW, nPbH, pred_idx);
1843         luma_mc(s, tmp2, tmpstride, ref1->frame,
1844                 &current_mv.mv[1], x0, y0, nPbW, nPbH, pred_idx);
1845
1846         if (weighted_pred) {
1847             s->hevcdsp.weighted_pred_avg[pred_idx](s->sh.luma_log2_weight_denom,
1848                                                    s->sh.luma_weight_l0[current_mv.ref_idx[0]],
1849                                                    s->sh.luma_weight_l1[current_mv.ref_idx[1]],
1850                                                    s->sh.luma_offset_l0[current_mv.ref_idx[0]],
1851                                                    s->sh.luma_offset_l1[current_mv.ref_idx[1]],
1852                                                    dst0, s->frame->linesize[0],
1853                                                    tmp, tmp2, tmpstride, nPbH);
1854         } else {
1855             s->hevcdsp.put_unweighted_pred_avg[pred_idx](dst0, s->frame->linesize[0],
1856                                                          tmp, tmp2, tmpstride, nPbH);
1857         }
1858
1859         chroma_mc(s, tmp, tmp2, tmpstride, ref0->frame,
1860                   &current_mv.mv[0], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1861         chroma_mc(s, tmp3, tmp4, tmpstride, ref1->frame,
1862                   &current_mv.mv[1], x0 / 2, y0 / 2, nPbW / 2, nPbH / 2, pred_idx);
1863
1864         if (weighted_pred) {
1865             s->hevcdsp.weighted_pred_avg_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1866                                                           s->sh.chroma_weight_l0[current_mv.ref_idx[0]][0],
1867                                                           s->sh.chroma_weight_l1[current_mv.ref_idx[1]][0],
1868                                                           s->sh.chroma_offset_l0[current_mv.ref_idx[0]][0],
1869                                                           s->sh.chroma_offset_l1[current_mv.ref_idx[1]][0],
1870                                                           dst1, s->frame->linesize[1], tmp, tmp3,
1871                                                           tmpstride, nPbH / 2);
1872             s->hevcdsp.weighted_pred_avg_chroma[pred_idx](s->sh.chroma_log2_weight_denom,
1873                                                           s->sh.chroma_weight_l0[current_mv.ref_idx[0]][1],
1874                                                           s->sh.chroma_weight_l1[current_mv.ref_idx[1]][1],
1875                                                           s->sh.chroma_offset_l0[current_mv.ref_idx[0]][1],
1876                                                           s->sh.chroma_offset_l1[current_mv.ref_idx[1]][1],
1877                                                           dst2, s->frame->linesize[2], tmp2, tmp4,
1878                                                           tmpstride, nPbH / 2);
1879         } else {
1880             s->hevcdsp.put_unweighted_pred_avg_chroma[pred_idx](dst1, s->frame->linesize[1], tmp, tmp3,  tmpstride, nPbH/2);
1881             s->hevcdsp.put_unweighted_pred_avg_chroma[pred_idx](dst2, s->frame->linesize[2], tmp2, tmp4, tmpstride, nPbH/2);
1882         }
1883     }
1884 }
1885
1886 /**
1887  * 8.4.1
1888  */
1889 static int luma_intra_pred_mode(HEVCContext *s, int x0, int y0, int pu_size,
1890                                 int prev_intra_luma_pred_flag)
1891 {
1892     HEVCLocalContext *lc = &s->HEVClc;
1893     int x_pu             = x0 >> s->ps.sps->log2_min_pu_size;
1894     int y_pu             = y0 >> s->ps.sps->log2_min_pu_size;
1895     int min_pu_width     = s->ps.sps->min_pu_width;
1896     int size_in_pus      = pu_size >> s->ps.sps->log2_min_pu_size;
1897     int x0b              = x0 & ((1 << s->ps.sps->log2_ctb_size) - 1);
1898     int y0b              = y0 & ((1 << s->ps.sps->log2_ctb_size) - 1);
1899
1900     int cand_up   = (lc->ctb_up_flag || y0b) ?
1901                     s->tab_ipm[(y_pu - 1) * min_pu_width + x_pu] : INTRA_DC;
1902     int cand_left = (lc->ctb_left_flag || x0b) ?
1903                     s->tab_ipm[y_pu * min_pu_width + x_pu - 1]   : INTRA_DC;
1904
1905     int y_ctb = (y0 >> (s->ps.sps->log2_ctb_size)) << (s->ps.sps->log2_ctb_size);
1906
1907     MvField *tab_mvf = s->ref->tab_mvf;
1908     int intra_pred_mode;
1909     int candidate[3];
1910     int i, j;
1911
1912     // intra_pred_mode prediction does not cross vertical CTB boundaries
1913     if ((y0 - 1) < y_ctb)
1914         cand_up = INTRA_DC;
1915
1916     if (cand_left == cand_up) {
1917         if (cand_left < 2) {
1918             candidate[0] = INTRA_PLANAR;
1919             candidate[1] = INTRA_DC;
1920             candidate[2] = INTRA_ANGULAR_26;
1921         } else {
1922             candidate[0] = cand_left;
1923             candidate[1] = 2 + ((cand_left - 2 - 1 + 32) & 31);
1924             candidate[2] = 2 + ((cand_left - 2 + 1) & 31);
1925         }
1926     } else {
1927         candidate[0] = cand_left;
1928         candidate[1] = cand_up;
1929         if (candidate[0] != INTRA_PLANAR && candidate[1] != INTRA_PLANAR) {
1930             candidate[2] = INTRA_PLANAR;
1931         } else if (candidate[0] != INTRA_DC && candidate[1] != INTRA_DC) {
1932             candidate[2] = INTRA_DC;
1933         } else {
1934             candidate[2] = INTRA_ANGULAR_26;
1935         }
1936     }
1937
1938     if (prev_intra_luma_pred_flag) {
1939         intra_pred_mode = candidate[lc->pu.mpm_idx];
1940     } else {
1941         if (candidate[0] > candidate[1])
1942             FFSWAP(uint8_t, candidate[0], candidate[1]);
1943         if (candidate[0] > candidate[2])
1944             FFSWAP(uint8_t, candidate[0], candidate[2]);
1945         if (candidate[1] > candidate[2])
1946             FFSWAP(uint8_t, candidate[1], candidate[2]);
1947
1948         intra_pred_mode = lc->pu.rem_intra_luma_pred_mode;
1949         for (i = 0; i < 3; i++)
1950             if (intra_pred_mode >= candidate[i])
1951                 intra_pred_mode++;
1952     }
1953
1954     /* write the intra prediction units into the mv array */
1955     if (!size_in_pus)
1956         size_in_pus = 1;
1957     for (i = 0; i < size_in_pus; i++) {
1958         memset(&s->tab_ipm[(y_pu + i) * min_pu_width + x_pu],
1959                intra_pred_mode, size_in_pus);
1960
1961         for (j = 0; j < size_in_pus; j++) {
1962             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].is_intra     = 1;
1963             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].pred_flag[0] = 0;
1964             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].pred_flag[1] = 0;
1965             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].ref_idx[0]   = 0;
1966             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].ref_idx[1]   = 0;
1967             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[0].x      = 0;
1968             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[0].y      = 0;
1969             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[1].x      = 0;
1970             tab_mvf[(y_pu + j) * min_pu_width + x_pu + i].mv[1].y      = 0;
1971         }
1972     }
1973
1974     return intra_pred_mode;
1975 }
1976
1977 static av_always_inline void set_ct_depth(HEVCContext *s, int x0, int y0,
1978                                           int log2_cb_size, int ct_depth)
1979 {
1980     int length = (1 << log2_cb_size) >> s->ps.sps->log2_min_cb_size;
1981     int x_cb   = x0 >> s->ps.sps->log2_min_cb_size;
1982     int y_cb   = y0 >> s->ps.sps->log2_min_cb_size;
1983     int y;
1984
1985     for (y = 0; y < length; y++)
1986         memset(&s->tab_ct_depth[(y_cb + y) * s->ps.sps->min_cb_width + x_cb],
1987                ct_depth, length);
1988 }
1989
1990 static void intra_prediction_unit(HEVCContext *s, int x0, int y0,
1991                                   int log2_cb_size)
1992 {
1993     HEVCLocalContext *lc = &s->HEVClc;
1994     static const uint8_t intra_chroma_table[4] = { 0, 26, 10, 1 };
1995     uint8_t prev_intra_luma_pred_flag[4];
1996     int split   = lc->cu.part_mode == PART_NxN;
1997     int pb_size = (1 << log2_cb_size) >> split;
1998     int side    = split + 1;
1999     int chroma_mode;
2000     int i, j;
2001
2002     for (i = 0; i < side; i++)
2003         for (j = 0; j < side; j++)
2004             prev_intra_luma_pred_flag[2 * i + j] = ff_hevc_prev_intra_luma_pred_flag_decode(s);
2005
2006     for (i = 0; i < side; i++) {
2007         for (j = 0; j < side; j++) {
2008             if (prev_intra_luma_pred_flag[2 * i + j])
2009                 lc->pu.mpm_idx = ff_hevc_mpm_idx_decode(s);
2010             else
2011                 lc->pu.rem_intra_luma_pred_mode = ff_hevc_rem_intra_luma_pred_mode_decode(s);
2012
2013             lc->pu.intra_pred_mode[2 * i + j] =
2014                 luma_intra_pred_mode(s, x0 + pb_size * j, y0 + pb_size * i, pb_size,
2015                                      prev_intra_luma_pred_flag[2 * i + j]);
2016         }
2017     }
2018
2019     chroma_mode = ff_hevc_intra_chroma_pred_mode_decode(s);
2020     if (chroma_mode != 4) {
2021         if (lc->pu.intra_pred_mode[0] == intra_chroma_table[chroma_mode])
2022             lc->pu.intra_pred_mode_c = 34;
2023         else
2024             lc->pu.intra_pred_mode_c = intra_chroma_table[chroma_mode];
2025     } else {
2026         lc->pu.intra_pred_mode_c = lc->pu.intra_pred_mode[0];
2027     }
2028 }
2029
2030 static void intra_prediction_unit_default_value(HEVCContext *s,
2031                                                 int x0, int y0,
2032                                                 int log2_cb_size)
2033 {
2034     HEVCLocalContext *lc = &s->HEVClc;
2035     int pb_size          = 1 << log2_cb_size;
2036     int size_in_pus      = pb_size >> s->ps.sps->log2_min_pu_size;
2037     int min_pu_width     = s->ps.sps->min_pu_width;
2038     MvField *tab_mvf     = s->ref->tab_mvf;
2039     int x_pu             = x0 >> s->ps.sps->log2_min_pu_size;
2040     int y_pu             = y0 >> s->ps.sps->log2_min_pu_size;
2041     int j, k;
2042
2043     if (size_in_pus == 0)
2044         size_in_pus = 1;
2045     for (j = 0; j < size_in_pus; j++) {
2046         memset(&s->tab_ipm[(y_pu + j) * min_pu_width + x_pu], INTRA_DC, size_in_pus);
2047         for (k = 0; k < size_in_pus; k++)
2048             tab_mvf[(y_pu + j) * min_pu_width + x_pu + k].is_intra = lc->cu.pred_mode == MODE_INTRA;
2049     }
2050 }
2051
2052 static int hls_coding_unit(HEVCContext *s, int x0, int y0, int log2_cb_size)
2053 {
2054     int cb_size          = 1 << log2_cb_size;
2055     HEVCLocalContext *lc = &s->HEVClc;
2056     int log2_min_cb_size = s->ps.sps->log2_min_cb_size;
2057     int length           = cb_size >> log2_min_cb_size;
2058     int min_cb_width     = s->ps.sps->min_cb_width;
2059     int x_cb             = x0 >> log2_min_cb_size;
2060     int y_cb             = y0 >> log2_min_cb_size;
2061     int x, y, ret;
2062
2063     lc->cu.x                = x0;
2064     lc->cu.y                = y0;
2065     lc->cu.pred_mode        = MODE_INTRA;
2066     lc->cu.part_mode        = PART_2Nx2N;
2067     lc->cu.intra_split_flag = 0;
2068
2069     SAMPLE_CTB(s->skip_flag, x_cb, y_cb) = 0;
2070     for (x = 0; x < 4; x++)
2071         lc->pu.intra_pred_mode[x] = 1;
2072     if (s->ps.pps->transquant_bypass_enable_flag) {
2073         lc->cu.cu_transquant_bypass_flag = ff_hevc_cu_transquant_bypass_flag_decode(s);
2074         if (lc->cu.cu_transquant_bypass_flag)
2075             set_deblocking_bypass(s, x0, y0, log2_cb_size);
2076     } else
2077         lc->cu.cu_transquant_bypass_flag = 0;
2078
2079     if (s->sh.slice_type != HEVC_SLICE_I) {
2080         uint8_t skip_flag = ff_hevc_skip_flag_decode(s, x0, y0, x_cb, y_cb);
2081
2082         x = y_cb * min_cb_width + x_cb;
2083         for (y = 0; y < length; y++) {
2084             memset(&s->skip_flag[x], skip_flag, length);
2085             x += min_cb_width;
2086         }
2087         lc->cu.pred_mode = skip_flag ? MODE_SKIP : MODE_INTER;
2088     }
2089
2090     if (SAMPLE_CTB(s->skip_flag, x_cb, y_cb)) {
2091         hls_prediction_unit(s, x0, y0, cb_size, cb_size, log2_cb_size, 0);
2092         intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2093
2094         if (!s->sh.disable_deblocking_filter_flag)
2095             ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
2096     } else {
2097         int pcm_flag = 0;
2098
2099         if (s->sh.slice_type != HEVC_SLICE_I)
2100             lc->cu.pred_mode = ff_hevc_pred_mode_decode(s);
2101         if (lc->cu.pred_mode != MODE_INTRA ||
2102             log2_cb_size == s->ps.sps->log2_min_cb_size) {
2103             lc->cu.part_mode        = ff_hevc_part_mode_decode(s, log2_cb_size);
2104             lc->cu.intra_split_flag = lc->cu.part_mode == PART_NxN &&
2105                                       lc->cu.pred_mode == MODE_INTRA;
2106         }
2107
2108         if (lc->cu.pred_mode == MODE_INTRA) {
2109             if (lc->cu.part_mode == PART_2Nx2N && s->ps.sps->pcm_enabled_flag &&
2110                 log2_cb_size >= s->ps.sps->pcm.log2_min_pcm_cb_size &&
2111                 log2_cb_size <= s->ps.sps->pcm.log2_max_pcm_cb_size) {
2112                 pcm_flag = ff_hevc_pcm_flag_decode(s);
2113             }
2114             if (pcm_flag) {
2115                 intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2116                 ret = hls_pcm_sample(s, x0, y0, log2_cb_size);
2117                 if (s->ps.sps->pcm.loop_filter_disable_flag)
2118                     set_deblocking_bypass(s, x0, y0, log2_cb_size);
2119
2120                 if (ret < 0)
2121                     return ret;
2122             } else {
2123                 intra_prediction_unit(s, x0, y0, log2_cb_size);
2124             }
2125         } else {
2126             intra_prediction_unit_default_value(s, x0, y0, log2_cb_size);
2127             switch (lc->cu.part_mode) {
2128             case PART_2Nx2N:
2129                 hls_prediction_unit(s, x0, y0, cb_size, cb_size, log2_cb_size, 0);
2130                 break;
2131             case PART_2NxN:
2132                 hls_prediction_unit(s, x0, y0,               cb_size, cb_size / 2, log2_cb_size, 0);
2133                 hls_prediction_unit(s, x0, y0 + cb_size / 2, cb_size, cb_size / 2, log2_cb_size, 1);
2134                 break;
2135             case PART_Nx2N:
2136                 hls_prediction_unit(s, x0,               y0, cb_size / 2, cb_size, log2_cb_size, 0);
2137                 hls_prediction_unit(s, x0 + cb_size / 2, y0, cb_size / 2, cb_size, log2_cb_size, 1);
2138                 break;
2139             case PART_2NxnU:
2140                 hls_prediction_unit(s, x0, y0,               cb_size, cb_size     / 4, log2_cb_size, 0);
2141                 hls_prediction_unit(s, x0, y0 + cb_size / 4, cb_size, cb_size * 3 / 4, log2_cb_size, 1);
2142                 break;
2143             case PART_2NxnD:
2144                 hls_prediction_unit(s, x0, y0,                   cb_size, cb_size * 3 / 4, log2_cb_size, 0);
2145                 hls_prediction_unit(s, x0, y0 + cb_size * 3 / 4, cb_size, cb_size     / 4, log2_cb_size, 1);
2146                 break;
2147             case PART_nLx2N:
2148                 hls_prediction_unit(s, x0,               y0, cb_size     / 4, cb_size, log2_cb_size, 0);
2149                 hls_prediction_unit(s, x0 + cb_size / 4, y0, cb_size * 3 / 4, cb_size, log2_cb_size, 1);
2150                 break;
2151             case PART_nRx2N:
2152                 hls_prediction_unit(s, x0,                   y0, cb_size * 3 / 4, cb_size, log2_cb_size, 0);
2153                 hls_prediction_unit(s, x0 + cb_size * 3 / 4, y0, cb_size     / 4, cb_size, log2_cb_size, 1);
2154                 break;
2155             case PART_NxN:
2156                 hls_prediction_unit(s, x0,               y0,               cb_size / 2, cb_size / 2, log2_cb_size, 0);
2157                 hls_prediction_unit(s, x0 + cb_size / 2, y0,               cb_size / 2, cb_size / 2, log2_cb_size, 1);
2158                 hls_prediction_unit(s, x0,               y0 + cb_size / 2, cb_size / 2, cb_size / 2, log2_cb_size, 2);
2159                 hls_prediction_unit(s, x0 + cb_size / 2, y0 + cb_size / 2, cb_size / 2, cb_size / 2, log2_cb_size, 3);
2160                 break;
2161             }
2162         }
2163
2164         if (!pcm_flag) {
2165             int rqt_root_cbf = 1;
2166
2167             if (lc->cu.pred_mode != MODE_INTRA &&
2168                 !(lc->cu.part_mode == PART_2Nx2N && lc->pu.merge_flag)) {
2169                 rqt_root_cbf = ff_hevc_no_residual_syntax_flag_decode(s);
2170             }
2171             if (rqt_root_cbf) {
2172                 lc->cu.max_trafo_depth = lc->cu.pred_mode == MODE_INTRA ?
2173                                          s->ps.sps->max_transform_hierarchy_depth_intra + lc->cu.intra_split_flag :
2174                                          s->ps.sps->max_transform_hierarchy_depth_inter;
2175                 ret = hls_transform_tree(s, x0, y0, x0, y0, x0, y0,
2176                                          log2_cb_size,
2177                                          log2_cb_size, 0, 0, 0, 0);
2178                 if (ret < 0)
2179                     return ret;
2180             } else {
2181                 if (!s->sh.disable_deblocking_filter_flag)
2182                     ff_hevc_deblocking_boundary_strengths(s, x0, y0, log2_cb_size);
2183             }
2184         }
2185     }
2186
2187     if (s->ps.pps->cu_qp_delta_enabled_flag && lc->tu.is_cu_qp_delta_coded == 0)
2188         ff_hevc_set_qPy(s, x0, y0, x0, y0, log2_cb_size);
2189
2190     x = y_cb * min_cb_width + x_cb;
2191     for (y = 0; y < length; y++) {
2192         memset(&s->qp_y_tab[x], lc->qp_y, length);
2193         x += min_cb_width;
2194     }
2195
2196     set_ct_depth(s, x0, y0, log2_cb_size, lc->ct.depth);
2197
2198     return 0;
2199 }
2200
2201 static int hls_coding_quadtree(HEVCContext *s, int x0, int y0,
2202                                int log2_cb_size, int cb_depth)
2203 {
2204     HEVCLocalContext *lc = &s->HEVClc;
2205     const int cb_size    = 1 << log2_cb_size;
2206     int split_cu;
2207
2208     lc->ct.depth = cb_depth;
2209     if (x0 + cb_size <= s->ps.sps->width  &&
2210         y0 + cb_size <= s->ps.sps->height &&
2211         log2_cb_size > s->ps.sps->log2_min_cb_size) {
2212         split_cu = ff_hevc_split_coding_unit_flag_decode(s, cb_depth, x0, y0);
2213     } else {
2214         split_cu = (log2_cb_size > s->ps.sps->log2_min_cb_size);
2215     }
2216     if (s->ps.pps->cu_qp_delta_enabled_flag &&
2217         log2_cb_size >= s->ps.sps->log2_ctb_size - s->ps.pps->diff_cu_qp_delta_depth) {
2218         lc->tu.is_cu_qp_delta_coded = 0;
2219         lc->tu.cu_qp_delta          = 0;
2220     }
2221
2222     if (split_cu) {
2223         const int cb_size_split = cb_size >> 1;
2224         const int x1 = x0 + cb_size_split;
2225         const int y1 = y0 + cb_size_split;
2226
2227         log2_cb_size--;
2228         cb_depth++;
2229
2230 #define SUBDIVIDE(x, y)                                                \
2231 do {                                                                   \
2232     if (x < s->ps.sps->width && y < s->ps.sps->height) {                     \
2233         int ret = hls_coding_quadtree(s, x, y, log2_cb_size, cb_depth);\
2234         if (ret < 0)                                                   \
2235             return ret;                                                \
2236     }                                                                  \
2237 } while (0)
2238
2239         SUBDIVIDE(x0, y0);
2240         SUBDIVIDE(x1, y0);
2241         SUBDIVIDE(x0, y1);
2242         SUBDIVIDE(x1, y1);
2243     } else {
2244         int ret = hls_coding_unit(s, x0, y0, log2_cb_size);
2245         if (ret < 0)
2246             return ret;
2247     }
2248
2249     return 0;
2250 }
2251
2252 static void hls_decode_neighbour(HEVCContext *s, int x_ctb, int y_ctb,
2253                                  int ctb_addr_ts)
2254 {
2255     HEVCLocalContext *lc  = &s->HEVClc;
2256     int ctb_size          = 1 << s->ps.sps->log2_ctb_size;
2257     int ctb_addr_rs       = s->ps.pps->ctb_addr_ts_to_rs[ctb_addr_ts];
2258     int ctb_addr_in_slice = ctb_addr_rs - s->sh.slice_addr;
2259
2260     s->tab_slice_address[ctb_addr_rs] = s->sh.slice_addr;
2261
2262     if (s->ps.pps->entropy_coding_sync_enabled_flag) {
2263         if (x_ctb == 0 && (y_ctb & (ctb_size - 1)) == 0)
2264             lc->first_qp_group = 1;
2265         lc->end_of_tiles_x = s->ps.sps->width;
2266     } else if (s->ps.pps->tiles_enabled_flag) {
2267         if (ctb_addr_ts && s->ps.pps->tile_id[ctb_addr_ts] != s->ps.pps->tile_id[ctb_addr_ts - 1]) {
2268             int idxX = s->ps.pps->col_idxX[x_ctb >> s->ps.sps->log2_ctb_size];
2269             lc->start_of_tiles_x = x_ctb;
2270             lc->end_of_tiles_x   = x_ctb + (s->ps.pps->column_width[idxX] << s->ps.sps->log2_ctb_size);
2271             lc->first_qp_group   = 1;
2272         }
2273     } else {
2274         lc->end_of_tiles_x = s->ps.sps->width;
2275     }
2276
2277     lc->end_of_tiles_y = FFMIN(y_ctb + ctb_size, s->ps.sps->height);
2278
2279     lc->boundary_flags = 0;
2280     if (s->ps.pps->tiles_enabled_flag) {
2281         if (x_ctb > 0 && s->ps.pps->tile_id[ctb_addr_ts] != s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs - 1]])
2282             lc->boundary_flags |= BOUNDARY_LEFT_TILE;
2283         if (x_ctb > 0 && s->tab_slice_address[ctb_addr_rs] != s->tab_slice_address[ctb_addr_rs - 1])
2284             lc->boundary_flags |= BOUNDARY_LEFT_SLICE;
2285         if (y_ctb > 0 && s->ps.pps->tile_id[ctb_addr_ts] != s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs - s->ps.sps->ctb_width]])
2286             lc->boundary_flags |= BOUNDARY_UPPER_TILE;
2287         if (y_ctb > 0 && s->tab_slice_address[ctb_addr_rs] != s->tab_slice_address[ctb_addr_rs - s->ps.sps->ctb_width])
2288             lc->boundary_flags |= BOUNDARY_UPPER_SLICE;
2289     } else {
2290         if (!ctb_addr_in_slice)
2291             lc->boundary_flags |= BOUNDARY_LEFT_SLICE;
2292         if (ctb_addr_in_slice < s->ps.sps->ctb_width)
2293             lc->boundary_flags |= BOUNDARY_UPPER_SLICE;
2294     }
2295
2296     lc->ctb_left_flag = ((x_ctb > 0) && (ctb_addr_in_slice > 0) && !(lc->boundary_flags & BOUNDARY_LEFT_TILE));
2297     lc->ctb_up_flag   = ((y_ctb > 0) && (ctb_addr_in_slice >= s->ps.sps->ctb_width) && !(lc->boundary_flags & BOUNDARY_UPPER_TILE));
2298     lc->ctb_up_right_flag = ((y_ctb > 0)  && (ctb_addr_in_slice+1 >= s->ps.sps->ctb_width) && (s->ps.pps->tile_id[ctb_addr_ts] == s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs+1 - s->ps.sps->ctb_width]]));
2299     lc->ctb_up_left_flag = ((x_ctb > 0) && (y_ctb > 0)  && (ctb_addr_in_slice-1 >= s->ps.sps->ctb_width) && (s->ps.pps->tile_id[ctb_addr_ts] == s->ps.pps->tile_id[s->ps.pps->ctb_addr_rs_to_ts[ctb_addr_rs-1 - s->ps.sps->ctb_width]]));
2300 }
2301
2302 static int hls_slice_data(HEVCContext *s)
2303 {
2304     int ctb_size    = 1 << s->ps.sps->log2_ctb_size;
2305     int more_data   = 1;
2306     int x_ctb       = 0;
2307     int y_ctb       = 0;
2308     int ctb_addr_ts = s->ps.pps->ctb_addr_rs_to_ts[s->sh.slice_ctb_addr_rs];
2309     int ret;
2310
2311     while (more_data && ctb_addr_ts < s->ps.sps->ctb_size) {
2312         int ctb_addr_rs = s->ps.pps->ctb_addr_ts_to_rs[ctb_addr_ts];
2313
2314         x_ctb = (ctb_addr_rs % ((s->ps.sps->width + ctb_size - 1) >> s->ps.sps->log2_ctb_size)) << s->ps.sps->log2_ctb_size;
2315         y_ctb = (ctb_addr_rs / ((s->ps.sps->width + ctb_size - 1) >> s->ps.sps->log2_ctb_size)) << s->ps.sps->log2_ctb_size;
2316         hls_decode_neighbour(s, x_ctb, y_ctb, ctb_addr_ts);
2317
2318         ff_hevc_cabac_init(s, ctb_addr_ts);
2319
2320         hls_sao_param(s, x_ctb >> s->ps.sps->log2_ctb_size, y_ctb >> s->ps.sps->log2_ctb_size);
2321
2322         s->deblock[ctb_addr_rs].beta_offset = s->sh.beta_offset;
2323         s->deblock[ctb_addr_rs].tc_offset   = s->sh.tc_offset;
2324         s->filter_slice_edges[ctb_addr_rs]  = s->sh.slice_loop_filter_across_slices_enabled_flag;
2325
2326         ret = hls_coding_quadtree(s, x_ctb, y_ctb, s->ps.sps->log2_ctb_size, 0);
2327         if (ret < 0)
2328             return ret;
2329         more_data = !ff_hevc_end_of_slice_flag_decode(s);
2330
2331         ctb_addr_ts++;
2332         ff_hevc_save_states(s, ctb_addr_ts);
2333         ff_hevc_hls_filters(s, x_ctb, y_ctb, ctb_size);
2334     }
2335
2336     if (x_ctb + ctb_size >= s->ps.sps->width &&
2337         y_ctb + ctb_size >= s->ps.sps->height)
2338         ff_hevc_hls_filter(s, x_ctb, y_ctb);
2339
2340     return ctb_addr_ts;
2341 }
2342
2343 static void restore_tqb_pixels(HEVCContext *s)
2344 {
2345     int min_pu_size = 1 << s->ps.sps->log2_min_pu_size;
2346     int x, y, c_idx;
2347
2348     for (c_idx = 0; c_idx < 3; c_idx++) {
2349         ptrdiff_t stride = s->frame->linesize[c_idx];
2350         int hshift       = s->ps.sps->hshift[c_idx];
2351         int vshift       = s->ps.sps->vshift[c_idx];
2352         for (y = 0; y < s->ps.sps->min_pu_height; y++) {
2353             for (x = 0; x < s->ps.sps->min_pu_width; x++) {
2354                 if (s->is_pcm[y * s->ps.sps->min_pu_width + x]) {
2355                     int n;
2356                     int len      = min_pu_size >> hshift;
2357                     uint8_t *src = &s->frame->data[c_idx][((y << s->ps.sps->log2_min_pu_size) >> vshift) * stride + (((x << s->ps.sps->log2_min_pu_size) >> hshift) << s->ps.sps->pixel_shift)];
2358                     uint8_t *dst = &s->sao_frame->data[c_idx][((y << s->ps.sps->log2_min_pu_size) >> vshift) * stride + (((x << s->ps.sps->log2_min_pu_size) >> hshift) << s->ps.sps->pixel_shift)];
2359                     for (n = 0; n < (min_pu_size >> vshift); n++) {
2360                         memcpy(dst, src, len);
2361                         src += stride;
2362                         dst += stride;
2363                     }
2364                 }
2365             }
2366         }
2367     }
2368 }
2369
2370 static int set_side_data(HEVCContext *s)
2371 {
2372     AVFrame *out = s->ref->frame;
2373
2374     if (s->sei.frame_packing.present &&
2375         s->sei.frame_packing.arrangement_type >= 3 &&
2376         s->sei.frame_packing.arrangement_type <= 5 &&
2377         s->sei.frame_packing.content_interpretation_type > 0 &&
2378         s->sei.frame_packing.content_interpretation_type < 3) {
2379         AVStereo3D *stereo = av_stereo3d_create_side_data(out);
2380         if (!stereo)
2381             return AVERROR(ENOMEM);
2382
2383         switch (s->sei.frame_packing.arrangement_type) {
2384         case 3:
2385             if (s->sei.frame_packing.quincunx_subsampling)
2386                 stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX;
2387             else
2388                 stereo->type = AV_STEREO3D_SIDEBYSIDE;
2389             break;
2390         case 4:
2391             stereo->type = AV_STEREO3D_TOPBOTTOM;
2392             break;
2393         case 5:
2394             stereo->type = AV_STEREO3D_FRAMESEQUENCE;
2395             break;
2396         }
2397
2398         if (s->sei.frame_packing.content_interpretation_type == 2)
2399             stereo->flags = AV_STEREO3D_FLAG_INVERT;
2400
2401         if (s->sei.frame_packing.arrangement_type == 5) {
2402             if (s->sei.frame_packing.current_frame_is_frame0_flag)
2403                 stereo->view = AV_STEREO3D_VIEW_LEFT;
2404             else
2405                 stereo->view = AV_STEREO3D_VIEW_RIGHT;
2406         }
2407     }
2408
2409     if (s->sei.display_orientation.present &&
2410         (s->sei.display_orientation.anticlockwise_rotation ||
2411          s->sei.display_orientation.hflip || s->sei.display_orientation.vflip)) {
2412         double angle = s->sei.display_orientation.anticlockwise_rotation * 360 / (double) (1 << 16);
2413         AVFrameSideData *rotation = av_frame_new_side_data(out,
2414                                                            AV_FRAME_DATA_DISPLAYMATRIX,
2415                                                            sizeof(int32_t) * 9);
2416         if (!rotation)
2417             return AVERROR(ENOMEM);
2418
2419         av_display_rotation_set((int32_t *)rotation->data, angle);
2420         av_display_matrix_flip((int32_t *)rotation->data,
2421                                s->sei.display_orientation.hflip,
2422                                s->sei.display_orientation.vflip);
2423     }
2424
2425     if (s->sei.alternative_transfer.present &&
2426         av_color_transfer_name(s->sei.alternative_transfer.preferred_transfer_characteristics) &&
2427         s->sei.alternative_transfer.preferred_transfer_characteristics != AVCOL_TRC_UNSPECIFIED) {
2428         s->avctx->color_trc = out->color_trc = s->sei.alternative_transfer.preferred_transfer_characteristics;
2429     }
2430
2431     return 0;
2432 }
2433
2434 static int hevc_frame_start(HEVCContext *s)
2435 {
2436     HEVCLocalContext *lc = &s->HEVClc;
2437     int ret;
2438
2439     memset(s->horizontal_bs, 0, 2 * s->bs_width * (s->bs_height + 1));
2440     memset(s->vertical_bs,   0, 2 * s->bs_width * (s->bs_height + 1));
2441     memset(s->cbf_luma,      0, s->ps.sps->min_tb_width * s->ps.sps->min_tb_height);
2442     memset(s->is_pcm,        0, s->ps.sps->min_pu_width * s->ps.sps->min_pu_height);
2443
2444     lc->start_of_tiles_x = 0;
2445     s->is_decoded        = 0;
2446     s->first_nal_type    = s->nal_unit_type;
2447
2448     if (s->ps.pps->tiles_enabled_flag)
2449         lc->end_of_tiles_x = s->ps.pps->column_width[0] << s->ps.sps->log2_ctb_size;
2450
2451     ret = ff_hevc_set_new_ref(s, s->ps.sps->sao_enabled ? &s->sao_frame : &s->frame,
2452                               s->poc);
2453     if (ret < 0)
2454         goto fail;
2455
2456     ret = ff_hevc_frame_rps(s);
2457     if (ret < 0) {
2458         av_log(s->avctx, AV_LOG_ERROR, "Error constructing the frame RPS.\n");
2459         goto fail;
2460     }
2461
2462     s->ref->frame->key_frame = IS_IRAP(s);
2463
2464     ret = set_side_data(s);
2465     if (ret < 0)
2466         goto fail;
2467
2468     av_frame_unref(s->output_frame);
2469     ret = ff_hevc_output_frame(s, s->output_frame, 0);
2470     if (ret < 0)
2471         goto fail;
2472
2473     ff_thread_finish_setup(s->avctx);
2474
2475     return 0;
2476
2477 fail:
2478     if (s->ref)
2479         ff_hevc_unref_frame(s, s->ref, ~0);
2480     s->ref = NULL;
2481     return ret;
2482 }
2483
2484 static int decode_nal_unit(HEVCContext *s, const H2645NAL *nal)
2485 {
2486     HEVCLocalContext *lc = &s->HEVClc;
2487     GetBitContext *gb    = &lc->gb;
2488     int ctb_addr_ts, ret;
2489
2490     *gb              = nal->gb;
2491     s->nal_unit_type = nal->type;
2492     s->temporal_id   = nal->temporal_id;
2493
2494     switch (s->nal_unit_type) {
2495     case HEVC_NAL_VPS:
2496         ret = ff_hevc_decode_nal_vps(gb, s->avctx, &s->ps);
2497         if (ret < 0)
2498             goto fail;
2499         break;
2500     case HEVC_NAL_SPS:
2501         ret = ff_hevc_decode_nal_sps(gb, s->avctx, &s->ps,
2502                                      s->apply_defdispwin);
2503         if (ret < 0)
2504             goto fail;
2505         break;
2506     case HEVC_NAL_PPS:
2507         ret = ff_hevc_decode_nal_pps(gb, s->avctx, &s->ps);
2508         if (ret < 0)
2509             goto fail;
2510         break;
2511     case HEVC_NAL_SEI_PREFIX:
2512     case HEVC_NAL_SEI_SUFFIX:
2513         ret = ff_hevc_decode_nal_sei(gb, s->avctx, &s->sei,
2514                                      s->nal_unit_type);
2515         if (ret < 0)
2516             goto fail;
2517         break;
2518     case HEVC_NAL_TRAIL_R:
2519     case HEVC_NAL_TRAIL_N:
2520     case HEVC_NAL_TSA_N:
2521     case HEVC_NAL_TSA_R:
2522     case HEVC_NAL_STSA_N:
2523     case HEVC_NAL_STSA_R:
2524     case HEVC_NAL_BLA_W_LP:
2525     case HEVC_NAL_BLA_W_RADL:
2526     case HEVC_NAL_BLA_N_LP:
2527     case HEVC_NAL_IDR_W_RADL:
2528     case HEVC_NAL_IDR_N_LP:
2529     case HEVC_NAL_CRA_NUT:
2530     case HEVC_NAL_RADL_N:
2531     case HEVC_NAL_RADL_R:
2532     case HEVC_NAL_RASL_N:
2533     case HEVC_NAL_RASL_R:
2534         ret = hls_slice_header(s);
2535         if (ret < 0)
2536             return ret;
2537
2538         if (s->max_ra == INT_MAX) {
2539             if (s->nal_unit_type == HEVC_NAL_CRA_NUT || IS_BLA(s)) {
2540                 s->max_ra = s->poc;
2541             } else {
2542                 if (IS_IDR(s))
2543                     s->max_ra = INT_MIN;
2544             }
2545         }
2546
2547         if ((s->nal_unit_type == HEVC_NAL_RASL_R || s->nal_unit_type == HEVC_NAL_RASL_N) &&
2548             s->poc <= s->max_ra) {
2549             s->is_decoded = 0;
2550             break;
2551         } else {
2552             if (s->nal_unit_type == HEVC_NAL_RASL_R && s->poc > s->max_ra)
2553                 s->max_ra = INT_MIN;
2554         }
2555
2556         if (s->sh.first_slice_in_pic_flag) {
2557             ret = hevc_frame_start(s);
2558             if (ret < 0)
2559                 return ret;
2560         } else if (!s->ref) {
2561             av_log(s->avctx, AV_LOG_ERROR, "First slice in a frame missing.\n");
2562             goto fail;
2563         }
2564
2565         if (s->nal_unit_type != s->first_nal_type) {
2566             av_log(s->avctx, AV_LOG_ERROR,
2567                    "Non-matching NAL types of the VCL NALUs: %d %d\n",
2568                    s->first_nal_type, s->nal_unit_type);
2569             return AVERROR_INVALIDDATA;
2570         }
2571
2572         if (!s->sh.dependent_slice_segment_flag &&
2573             s->sh.slice_type != HEVC_SLICE_I) {
2574             ret = ff_hevc_slice_rpl(s);
2575             if (ret < 0) {
2576                 av_log(s->avctx, AV_LOG_WARNING,
2577                        "Error constructing the reference lists for the current slice.\n");
2578                 goto fail;
2579             }
2580         }
2581
2582         if (s->sh.first_slice_in_pic_flag && s->avctx->hwaccel) {
2583             ret = s->avctx->hwaccel->start_frame(s->avctx, NULL, 0);
2584             if (ret < 0)
2585                 goto fail;
2586         }
2587
2588         if (s->avctx->hwaccel) {
2589             ret = s->avctx->hwaccel->decode_slice(s->avctx, nal->raw_data, nal->raw_size);
2590             if (ret < 0)
2591                 goto fail;
2592         } else {
2593             ctb_addr_ts = hls_slice_data(s);
2594             if (ctb_addr_ts >= (s->ps.sps->ctb_width * s->ps.sps->ctb_height)) {
2595                 s->is_decoded = 1;
2596                 if ((s->ps.pps->transquant_bypass_enable_flag ||
2597                      (s->ps.sps->pcm.loop_filter_disable_flag && s->ps.sps->pcm_enabled_flag)) &&
2598                     s->ps.sps->sao_enabled)
2599                     restore_tqb_pixels(s);
2600             }
2601
2602             if (ctb_addr_ts < 0) {
2603                 ret = ctb_addr_ts;
2604                 goto fail;
2605             }
2606         }
2607         break;
2608     case HEVC_NAL_EOS_NUT:
2609     case HEVC_NAL_EOB_NUT:
2610         s->seq_decode = (s->seq_decode + 1) & 0xff;
2611         s->max_ra     = INT_MAX;
2612         break;
2613     case HEVC_NAL_AUD:
2614     case HEVC_NAL_FD_NUT:
2615         break;
2616     default:
2617         av_log(s->avctx, AV_LOG_INFO,
2618                "Skipping NAL unit %d\n", s->nal_unit_type);
2619     }
2620
2621     return 0;
2622 fail:
2623     if (s->avctx->err_recognition & AV_EF_EXPLODE)
2624         return ret;
2625     return 0;
2626 }
2627
2628 static int decode_nal_units(HEVCContext *s, const uint8_t *buf, int length)
2629 {
2630     int i, ret = 0;
2631
2632     s->ref = NULL;
2633     s->eos = 0;
2634
2635     /* split the input packet into NAL units, so we know the upper bound on the
2636      * number of slices in the frame */
2637     ret = ff_h2645_packet_split(&s->pkt, buf, length, s->avctx, s->is_nalff,
2638                                 s->nal_length_size, s->avctx->codec_id);
2639     if (ret < 0) {
2640         av_log(s->avctx, AV_LOG_ERROR,
2641                "Error splitting the input into NAL units.\n");
2642         return ret;
2643     }
2644
2645     for (i = 0; i < s->pkt.nb_nals; i++) {
2646         if (s->pkt.nals[i].type == HEVC_NAL_EOB_NUT ||
2647             s->pkt.nals[i].type == HEVC_NAL_EOS_NUT)
2648             s->eos = 1;
2649     }
2650
2651     /* decode the NAL units */
2652     for (i = 0; i < s->pkt.nb_nals; i++) {
2653         ret = decode_nal_unit(s, &s->pkt.nals[i]);
2654         if (ret < 0) {
2655             av_log(s->avctx, AV_LOG_WARNING,
2656                    "Error parsing NAL unit #%d.\n", i);
2657             goto fail;
2658         }
2659     }
2660
2661 fail:
2662     if (s->ref)
2663         ff_thread_report_progress(&s->ref->tf, INT_MAX, 0);
2664
2665     return ret;
2666 }
2667
2668 static void print_md5(void *log_ctx, int level, uint8_t md5[16])
2669 {
2670     int i;
2671     for (i = 0; i < 16; i++)
2672         av_log(log_ctx, level, "%02"PRIx8, md5[i]);
2673 }
2674
2675 static int verify_md5(HEVCContext *s, AVFrame *frame)
2676 {
2677     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(frame->format);
2678     int pixel_shift;
2679     int i, j;
2680
2681     if (!desc)
2682         return AVERROR(EINVAL);
2683
2684     pixel_shift = desc->comp[0].depth > 8;
2685
2686     av_log(s->avctx, AV_LOG_DEBUG, "Verifying checksum for frame with POC %d: ",
2687            s->poc);
2688
2689     /* the checksums are LE, so we have to byteswap for >8bpp formats
2690      * on BE arches */
2691 #if HAVE_BIGENDIAN
2692     if (pixel_shift && !s->checksum_buf) {
2693         av_fast_malloc(&s->checksum_buf, &s->checksum_buf_size,
2694                        FFMAX3(frame->linesize[0], frame->linesize[1],
2695                               frame->linesize[2]));
2696         if (!s->checksum_buf)
2697             return AVERROR(ENOMEM);
2698     }
2699 #endif
2700
2701     for (i = 0; frame->data[i]; i++) {
2702         int width  = s->avctx->coded_width;
2703         int height = s->avctx->coded_height;
2704         int w = (i == 1 || i == 2) ? (width  >> desc->log2_chroma_w) : width;
2705         int h = (i == 1 || i == 2) ? (height >> desc->log2_chroma_h) : height;
2706         uint8_t md5[16];
2707
2708         av_md5_init(s->md5_ctx);
2709         for (j = 0; j < h; j++) {
2710             const uint8_t *src = frame->data[i] + j * frame->linesize[i];
2711 #if HAVE_BIGENDIAN
2712             if (pixel_shift) {
2713                 s->bdsp.bswap16_buf((uint16_t *) s->checksum_buf,
2714                                     (const uint16_t *) src, w);
2715                 src = s->checksum_buf;
2716             }
2717 #endif
2718             av_md5_update(s->md5_ctx, src, w << pixel_shift);
2719         }
2720         av_md5_final(s->md5_ctx, md5);
2721
2722         if (!memcmp(md5, s->sei.picture_hash.md5[i], 16)) {
2723             av_log   (s->avctx, AV_LOG_DEBUG, "plane %d - correct ", i);
2724             print_md5(s->avctx, AV_LOG_DEBUG, md5);
2725             av_log   (s->avctx, AV_LOG_DEBUG, "; ");
2726         } else {
2727             av_log   (s->avctx, AV_LOG_ERROR, "mismatching checksum of plane %d - ", i);
2728             print_md5(s->avctx, AV_LOG_ERROR, md5);
2729             av_log   (s->avctx, AV_LOG_ERROR, " != ");
2730             print_md5(s->avctx, AV_LOG_ERROR, s->sei.picture_hash.md5[i]);
2731             av_log   (s->avctx, AV_LOG_ERROR, "\n");
2732             return AVERROR_INVALIDDATA;
2733         }
2734     }
2735
2736     av_log(s->avctx, AV_LOG_DEBUG, "\n");
2737
2738     return 0;
2739 }
2740
2741 static int hevc_decode_extradata(HEVCContext *s, uint8_t *buf, int length)
2742 {
2743     AVCodecContext *avctx = s->avctx;
2744     GetByteContext gb;
2745     int ret, i;
2746
2747     bytestream2_init(&gb, buf, length);
2748
2749     if (length > 3 && (buf[0] || buf[1] || buf[2] > 1)) {
2750         /* It seems the extradata is encoded as hvcC format.
2751          * Temporarily, we support configurationVersion==0 until 14496-15 3rd
2752          * is finalized. When finalized, configurationVersion will be 1 and we
2753          * can recognize hvcC by checking if avctx->extradata[0]==1 or not. */
2754         int i, j, num_arrays, nal_len_size;
2755
2756         s->is_nalff = 1;
2757
2758         bytestream2_skip(&gb, 21);
2759         nal_len_size = (bytestream2_get_byte(&gb) & 3) + 1;
2760         num_arrays   = bytestream2_get_byte(&gb);
2761
2762         /* nal units in the hvcC always have length coded with 2 bytes,
2763          * so put a fake nal_length_size = 2 while parsing them */
2764         s->nal_length_size = 2;
2765
2766         /* Decode nal units from hvcC. */
2767         for (i = 0; i < num_arrays; i++) {
2768             int type = bytestream2_get_byte(&gb) & 0x3f;
2769             int cnt  = bytestream2_get_be16(&gb);
2770
2771             for (j = 0; j < cnt; j++) {
2772                 // +2 for the nal size field
2773                 int nalsize = bytestream2_peek_be16(&gb) + 2;
2774                 if (bytestream2_get_bytes_left(&gb) < nalsize) {
2775                     av_log(s->avctx, AV_LOG_ERROR,
2776                            "Invalid NAL unit size in extradata.\n");
2777                     return AVERROR_INVALIDDATA;
2778                 }
2779
2780                 ret = decode_nal_units(s, gb.buffer, nalsize);
2781                 if (ret < 0) {
2782                     av_log(avctx, AV_LOG_ERROR,
2783                            "Decoding nal unit %d %d from hvcC failed\n",
2784                            type, i);
2785                     return ret;
2786                 }
2787                 bytestream2_skip(&gb, nalsize);
2788             }
2789         }
2790
2791         /* Now store right nal length size, that will be used to parse
2792          * all other nals */
2793         s->nal_length_size = nal_len_size;
2794     } else {
2795         s->is_nalff = 0;
2796         ret = decode_nal_units(s, buf, length);
2797         if (ret < 0)
2798             return ret;
2799     }
2800
2801     /* export stream parameters from the first SPS */
2802     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.sps_list); i++) {
2803         if (s->ps.sps_list[i]) {
2804             const HEVCSPS *sps = (const HEVCSPS*)s->ps.sps_list[i]->data;
2805             export_stream_params(s->avctx, &s->ps, sps);
2806             break;
2807         }
2808     }
2809
2810     return 0;
2811 }
2812
2813 static int hevc_decode_frame(AVCodecContext *avctx, void *data, int *got_output,
2814                              AVPacket *avpkt)
2815 {
2816     int ret;
2817     int new_extradata_size;
2818     uint8_t *new_extradata;
2819     HEVCContext *s = avctx->priv_data;
2820
2821     if (!avpkt->size) {
2822         ret = ff_hevc_output_frame(s, data, 1);
2823         if (ret < 0)
2824             return ret;
2825
2826         *got_output = ret;
2827         return 0;
2828     }
2829
2830     new_extradata = av_packet_get_side_data(avpkt, AV_PKT_DATA_NEW_EXTRADATA,
2831                                             &new_extradata_size);
2832     if (new_extradata && new_extradata_size > 0) {
2833         ret = hevc_decode_extradata(s, new_extradata, new_extradata_size);
2834         if (ret < 0)
2835             return ret;
2836     }
2837
2838     s->ref = NULL;
2839     ret    = decode_nal_units(s, avpkt->data, avpkt->size);
2840     if (ret < 0)
2841         return ret;
2842
2843     if (avctx->hwaccel) {
2844         if (s->ref && avctx->hwaccel->end_frame(avctx) < 0)
2845             av_log(avctx, AV_LOG_ERROR,
2846                    "hardware accelerator failed to decode picture\n");
2847     } else {
2848         /* verify the SEI checksum */
2849         if (avctx->err_recognition & AV_EF_CRCCHECK && s->is_decoded &&
2850             s->sei.picture_hash.is_md5) {
2851             ret = verify_md5(s, s->ref->frame);
2852             if (ret < 0 && avctx->err_recognition & AV_EF_EXPLODE) {
2853                 ff_hevc_unref_frame(s, s->ref, ~0);
2854                 return ret;
2855             }
2856         }
2857     }
2858     s->sei.picture_hash.is_md5 = 0;
2859
2860     if (s->is_decoded) {
2861         av_log(avctx, AV_LOG_DEBUG, "Decoded frame with POC %d.\n", s->poc);
2862         s->is_decoded = 0;
2863     }
2864
2865     if (s->output_frame->buf[0]) {
2866         av_frame_move_ref(data, s->output_frame);
2867         *got_output = 1;
2868     }
2869
2870     return avpkt->size;
2871 }
2872
2873 static int hevc_ref_frame(HEVCContext *s, HEVCFrame *dst, HEVCFrame *src)
2874 {
2875     int ret = ff_thread_ref_frame(&dst->tf, &src->tf);
2876     if (ret < 0)
2877         return ret;
2878
2879     dst->tab_mvf_buf = av_buffer_ref(src->tab_mvf_buf);
2880     if (!dst->tab_mvf_buf)
2881         goto fail;
2882     dst->tab_mvf = src->tab_mvf;
2883
2884     dst->rpl_tab_buf = av_buffer_ref(src->rpl_tab_buf);
2885     if (!dst->rpl_tab_buf)
2886         goto fail;
2887     dst->rpl_tab = src->rpl_tab;
2888
2889     dst->rpl_buf = av_buffer_ref(src->rpl_buf);
2890     if (!dst->rpl_buf)
2891         goto fail;
2892
2893     dst->poc        = src->poc;
2894     dst->ctb_count  = src->ctb_count;
2895     dst->flags      = src->flags;
2896     dst->sequence   = src->sequence;
2897
2898     if (src->hwaccel_picture_private) {
2899         dst->hwaccel_priv_buf = av_buffer_ref(src->hwaccel_priv_buf);
2900         if (!dst->hwaccel_priv_buf)
2901             goto fail;
2902         dst->hwaccel_picture_private = dst->hwaccel_priv_buf->data;
2903     }
2904
2905     return 0;
2906 fail:
2907     ff_hevc_unref_frame(s, dst, ~0);
2908     return AVERROR(ENOMEM);
2909 }
2910
2911 static av_cold int hevc_decode_free(AVCodecContext *avctx)
2912 {
2913     HEVCContext       *s = avctx->priv_data;
2914     int i;
2915
2916     pic_arrays_free(s);
2917
2918     av_freep(&s->md5_ctx);
2919
2920     av_frame_free(&s->tmp_frame);
2921     av_frame_free(&s->output_frame);
2922
2923     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2924         ff_hevc_unref_frame(s, &s->DPB[i], ~0);
2925         av_frame_free(&s->DPB[i].frame);
2926     }
2927
2928     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.vps_list); i++)
2929         av_buffer_unref(&s->ps.vps_list[i]);
2930     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.sps_list); i++)
2931         av_buffer_unref(&s->ps.sps_list[i]);
2932     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.pps_list); i++)
2933         av_buffer_unref(&s->ps.pps_list[i]);
2934
2935     ff_h2645_packet_uninit(&s->pkt);
2936
2937     return 0;
2938 }
2939
2940 static av_cold int hevc_init_context(AVCodecContext *avctx)
2941 {
2942     HEVCContext *s = avctx->priv_data;
2943     int i;
2944
2945     s->avctx = avctx;
2946
2947     s->tmp_frame = av_frame_alloc();
2948     if (!s->tmp_frame)
2949         goto fail;
2950
2951     s->output_frame = av_frame_alloc();
2952     if (!s->output_frame)
2953         goto fail;
2954
2955     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2956         s->DPB[i].frame = av_frame_alloc();
2957         if (!s->DPB[i].frame)
2958             goto fail;
2959         s->DPB[i].tf.f = s->DPB[i].frame;
2960     }
2961
2962     s->max_ra = INT_MAX;
2963
2964     s->md5_ctx = av_md5_alloc();
2965     if (!s->md5_ctx)
2966         goto fail;
2967
2968     ff_bswapdsp_init(&s->bdsp);
2969
2970     s->context_initialized = 1;
2971
2972     return 0;
2973
2974 fail:
2975     hevc_decode_free(avctx);
2976     return AVERROR(ENOMEM);
2977 }
2978
2979 static int hevc_update_thread_context(AVCodecContext *dst,
2980                                       const AVCodecContext *src)
2981 {
2982     HEVCContext *s  = dst->priv_data;
2983     HEVCContext *s0 = src->priv_data;
2984     int i, ret;
2985
2986     if (!s->context_initialized) {
2987         ret = hevc_init_context(dst);
2988         if (ret < 0)
2989             return ret;
2990     }
2991
2992     for (i = 0; i < FF_ARRAY_ELEMS(s->DPB); i++) {
2993         ff_hevc_unref_frame(s, &s->DPB[i], ~0);
2994         if (s0->DPB[i].frame->buf[0]) {
2995             ret = hevc_ref_frame(s, &s->DPB[i], &s0->DPB[i]);
2996             if (ret < 0)
2997                 return ret;
2998         }
2999     }
3000
3001     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.vps_list); i++) {
3002         av_buffer_unref(&s->ps.vps_list[i]);
3003         if (s0->ps.vps_list[i]) {
3004             s->ps.vps_list[i] = av_buffer_ref(s0->ps.vps_list[i]);
3005             if (!s->ps.vps_list[i])
3006                 return AVERROR(ENOMEM);
3007         }
3008     }
3009
3010     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.sps_list); i++) {
3011         av_buffer_unref(&s->ps.sps_list[i]);
3012         if (s0->ps.sps_list[i]) {
3013             s->ps.sps_list[i] = av_buffer_ref(s0->ps.sps_list[i]);
3014             if (!s->ps.sps_list[i])
3015                 return AVERROR(ENOMEM);
3016         }
3017     }
3018
3019     for (i = 0; i < FF_ARRAY_ELEMS(s->ps.pps_list); i++) {
3020         av_buffer_unref(&s->ps.pps_list[i]);
3021         if (s0->ps.pps_list[i]) {
3022             s->ps.pps_list[i] = av_buffer_ref(s0->ps.pps_list[i]);
3023             if (!s->ps.pps_list[i])
3024                 return AVERROR(ENOMEM);
3025         }
3026     }
3027
3028     if (s->ps.sps != s0->ps.sps)
3029         ret = set_sps(s, s0->ps.sps, src->pix_fmt);
3030
3031     s->seq_decode = s0->seq_decode;
3032     s->seq_output = s0->seq_output;
3033     s->pocTid0    = s0->pocTid0;
3034     s->max_ra     = s0->max_ra;
3035
3036     s->is_nalff        = s0->is_nalff;
3037     s->nal_length_size = s0->nal_length_size;
3038
3039     if (s0->eos) {
3040         s->seq_decode = (s->seq_decode + 1) & 0xff;
3041         s->max_ra = INT_MAX;
3042     }
3043
3044     return 0;
3045 }
3046
3047 static av_cold int hevc_decode_init(AVCodecContext *avctx)
3048 {
3049     HEVCContext *s = avctx->priv_data;
3050     int ret;
3051
3052     avctx->internal->allocate_progress = 1;
3053
3054     ret = hevc_init_context(avctx);
3055     if (ret < 0)
3056         return ret;
3057
3058     if (avctx->extradata_size > 0 && avctx->extradata) {
3059         ret = hevc_decode_extradata(s, avctx->extradata, avctx->extradata_size);
3060         if (ret < 0) {
3061             hevc_decode_free(avctx);
3062             return ret;
3063         }
3064     }
3065
3066     return 0;
3067 }
3068
3069 static av_cold int hevc_init_thread_copy(AVCodecContext *avctx)
3070 {
3071     HEVCContext *s = avctx->priv_data;
3072     int ret;
3073
3074     memset(s, 0, sizeof(*s));
3075
3076     ret = hevc_init_context(avctx);
3077     if (ret < 0)
3078         return ret;
3079
3080     return 0;
3081 }
3082
3083 static void hevc_decode_flush(AVCodecContext *avctx)
3084 {
3085     HEVCContext *s = avctx->priv_data;
3086     ff_hevc_flush_dpb(s);
3087     s->max_ra = INT_MAX;
3088 }
3089
3090 #define OFFSET(x) offsetof(HEVCContext, x)
3091 #define PAR (AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
3092
3093 static const AVOption options[] = {
3094     { "apply_defdispwin", "Apply default display window from VUI", OFFSET(apply_defdispwin),
3095         AV_OPT_TYPE_INT, {.i64 = 0}, 0, 1, PAR },
3096     { NULL },
3097 };
3098
3099 static const AVClass hevc_decoder_class = {
3100     .class_name = "HEVC decoder",
3101     .item_name  = av_default_item_name,
3102     .option     = options,
3103     .version    = LIBAVUTIL_VERSION_INT,
3104 };
3105
3106 AVCodec ff_hevc_decoder = {
3107     .name                  = "hevc",
3108     .long_name             = NULL_IF_CONFIG_SMALL("HEVC (High Efficiency Video Coding)"),
3109     .type                  = AVMEDIA_TYPE_VIDEO,
3110     .id                    = AV_CODEC_ID_HEVC,
3111     .priv_data_size        = sizeof(HEVCContext),
3112     .priv_class            = &hevc_decoder_class,
3113     .init                  = hevc_decode_init,
3114     .close                 = hevc_decode_free,
3115     .decode                = hevc_decode_frame,
3116     .flush                 = hevc_decode_flush,
3117     .update_thread_context = hevc_update_thread_context,
3118     .init_thread_copy      = hevc_init_thread_copy,
3119     .capabilities          = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY |
3120                              AV_CODEC_CAP_FRAME_THREADS,
3121     .profiles              = NULL_IF_CONFIG_SMALL(ff_hevc_profiles),
3122     .caps_internal         = FF_CODEC_CAP_EXPORTS_CROPPING | FF_CODEC_CAP_INIT_THREADSAFE,
3123 };